一、安装apache2¶
# 10.0.8.1
1. 安装apache2¶
# 安装apache2!
apt install apache2 -y
2. 修改apache2配置文件¶
1. 查看配置文件¶
ls /etc/apache2/
root@ali:~# ls /etc/apache2/
apache2.conf conf-available conf-enabled envvars magic mods-available mods-enabled ports.conf sites-available sites-enabled
2. 修改配置文件¶
1. 添加自定义json日志格式!¶
# 添加自定义json日志格式
cd /etc/apache2/
vim apache2.conf
#1.在216行后面,新加自定义json日志格式!
LogFormat "{ \
\"@timestamp\": \"%{%Y-%m-%dT%H:%M:%S%z}t\", \
\"@version\": \"1\", \
\"tags\":[\"apache\"], \
\"message\": \"%h %l %u %t \\\"%r\\\" %>s %b\", \
\"clientip\": \"%a\", \
\"duration\": %D, \
\"status\": %>s, \
\"request\": \"%U%q\", \
\"urlpath\": \"%U\", \
\"urlquery\": \"%q\", \
\"bytes\": %B, \
\"method\": \"%m\", \
\"site\": \"%{Host}i\", \
\"referer\": \"%{Referer}i\", \
\"useragent\": \"%{User-agent}i\" \
}" apache_json
201 # The following directives define some format nicknames for use with
202 # a CustomLog directive.
203 #
204 # These deviate from the Common Log Format definitions in that they use %O
205 # (the actual bytes sent including headers) instead of %b (the size of the
206 # requested file), because the latter makes it impossible to detect partial
207 # requests.
208 #
209 # Note that the use of %{X-Forwarded-For}i instead of %h is not recommended.
210 # Use mod_remoteip instead.
211 #
212 LogFormat "%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" vhost_combined
213 LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" combined
214 LogFormat "%h %l %u %t \"%r\" %>s %O" common
215 LogFormat "%{Referer}i -> %U" referer
216 LogFormat "%{User-agent}i" agent
217
218 LogFormat "{ \
219 \"@timestamp\": \"%{%Y-%m-%dT%H:%M:%S%z}t\", \
220 \"@version\": \"1\", \
221 \"tags\":[\"apache\"], \
222 \"message\": \"%h %l %u %t \\\"%r\\\" %>s %b\", \
223 \"clientip\": \"%a\", \
224 \"duration\": %D, \
225 \"status\": %>s, \
226 \"request\": \"%U%q\", \
227 \"urlpath\": \"%U\", \
228 \"urlquery\": \"%q\", \
229 \"bytes\": %B, \
230 \"method\": \"%m\", \
231 \"site\": \"%{Host}i\", \
232 \"referer\": \"%{Referer}i\", \
233 \"useragent\": \"%{User-agent}i\" \
234 }" apache_json
235
2. 访问日志采用json格式¶
root@ali:/etc/apache2# cd sites-enabled/
root@ali:/etc/apache2/sites-enabled# ls
000-default.conf
root@ali:/etc/apache2/sites-enabled# vim 000-default.conf
# 修改21行,将combined改为apache_json,使用json格式记录访问日志!
1 <VirtualHost *:80>
2 # The ServerName directive sets the request scheme, hostname and port that
3 # the server uses to identify itself. This is used when creating
4 # redirection URLs. In the context of virtual hosts, the ServerName
5 # specifies what hostname must appear in the request's Host: header to
6 # match this virtual host. For the default virtual host (this file) this
7 # value is not decisive as it is used as a last resort host regardless.
8 # However, you must set it for any further virtual host explicitly.
9 #ServerName www.example.com
10
11 ServerAdmin webmaster@localhost
12 DocumentRoot /var/www/html
13
14 # Available loglevels: trace8, ..., trace1, debug, info, notice, warn,
15 # error, crit, alert, emerg.
16 # It is also possible to configure the loglevel for particular
17 # modules, e.g.
18 #LogLevel info ssl:warn
19
20 ErrorLog ${APACHE_LOG_DIR}/error.log
21 CustomLog ${APACHE_LOG_DIR}/access.log apache_json
22
23 # For most configuration files from conf-available/, which are
24 # enabled or disabled at a global level, it is possible to
25 # include a line for only one particular virtual host. For example the
26 # following line enables the CGI configuration for this host only
27 # after it has been globally disabled with "a2disconf".
28 #Include conf-available/serve-cgi-bin.conf
29 </VirtualHost>
30
31 # vim: syntax=apache ts=4 sw=4 sts=4 sr noet
3. 查看访问日志文件路径¶
cat envvars -n
# 找到:APACHE_LOG_DIR=/var/log/apache2$SUFFIX ,最终找到路径如下!
/var/log/apache2/access.log
root@ali:/etc/apache2# cat envvars -n
1 # envvars - default environment variables for apache2ctl
2
3 # this won't be correct after changing uid
4 unset HOME
5
6 # for supporting multiple apache2 instances
7 if [ "${APACHE_CONFDIR##/etc/apache2-}" != "${APACHE_CONFDIR}" ] ; then
8 SUFFIX="-${APACHE_CONFDIR##/etc/apache2-}"
9 else
10 SUFFIX=
11 fi
12
13 # Since there is no sane way to get the parsed apache2 config in scripts, some
14 # settings are defined via environment variables and then used in apache2ctl,
15 # /etc/init.d/apache2, /etc/logrotate.d/apache2, etc.
16 export APACHE_RUN_USER=www-data
17 export APACHE_RUN_GROUP=www-data
18 # temporary state file location. This might be changed to /run in Wheezy+1
19 export APACHE_PID_FILE=/var/run/apache2$SUFFIX/apache2.pid
20 export APACHE_RUN_DIR=/var/run/apache2$SUFFIX
21 export APACHE_LOCK_DIR=/var/lock/apache2$SUFFIX
22 # Only /var/log/apache2 is handled by /etc/logrotate.d/apache2.
23 export APACHE_LOG_DIR=/var/log/apache2$SUFFIX
24
25 ## The locale used by some modules like mod_dav
26 export LANG=C
27 ## Uncomment the following line to use the system default locale instead:
28 #. /etc/default/locale
29
30 export LANG
31
32 ## The command to get the status for 'apache2ctl status'.
33 ## Some packages providing 'www-browser' need '--dump' instead of '-dump'.
34 #export APACHE_LYNX='www-browser -dump'
35
36 ## If you need a higher file descriptor limit, uncomment and adjust the
37 ## following line (default is 8192):
38 #APACHE_ULIMIT_MAX_FILES='ulimit -n 65536'
39
40 ## If you would like to pass arguments to the web server, add them below
41 ## to the APACHE_ARGUMENTS environment.
42 #export APACHE_ARGUMENTS=''
43
44 ## Enable the debug mode for maintainer scripts.
45 ## This will produce a verbose output on package installations of web server modules and web application
46 ## installations which interact with Apache
47 #export APACHE2_MAINTSCRIPT_DEBUG=1
root@ali:/etc/apache2# ls /var/log/apache2/
access.log error.log other_vhosts_access.log
补充:各个变量代表的含义¶
# %a: 远程IP地址
# %A: 本地IP地址
# %B: 已发送的字节数,不包含HTTP头
`# %b: CLF格式的已发送字节数量,不包含HTTP头。例如当没有发送数据时,写入‘-’而不是0。
# %{FOOBAR}e: 环境变量FOOBAR的内容
# %f: 文件名字
`# %h: 远程主机
# %H 请求的协议
# %{Foobar}i: Foobar的内容,发送给服务器的请求的标头行。
`# %l: 远程登录名字(来自identd,如提供的话)
# %m: 请求的方法
# %{Foobar}n: 来自另外一个模块的注解“Foobar”的内容
# %{Foobar}o: Foobar的内容,应答的标头行
# %p: 服务器响应请求时使用的端口
# %P: 响应请求的子进程ID。
`# %q: 查询字符串(如果存在查询字符串,则包含“?”后面的部分;否则,它是一个空字符串。)
`# %r: 请求的第一行
# %s: 状态。对于进行内部重定向的请求,这是指*原来*请求的状态。如果用%...>s,则是指后来的请求。
`# %t: 以公共日志时间格式表示的时间(或称为标准英文格式)
# %{format}t: 以指定格式format表示的时间
# %T: 为响应请求而耗费的时间,以秒计
`# %u: 远程用户(来自auth;如果返回状态(%s)是401则可能是伪造的)
`# %U: 用户所请求的URL路径
# %v: 响应请求的服务器的ServerName
# %V: 依照UseCanonicalName设置得到的服务器名字
3. 启动apache2¶
systemctl start apache2
netstat -tunlp|grep 80
4. 监控访问日志¶
tail -f /var/log/apache2/access.log
# 发现日志确实已经变为json格式了!!!
root@ali:/etc/apache2# tail -f /var/log/apache2/access.log
{ "@timestamp": "2021-12-31T14:28:49+0800", "@version": "1", "tags":["apache"], "message": "52.235.35.202 - - [31/Dec/2021:14:28:49 +0800] \"GET /.env HTTP/1.1\" 404 275", "clientip": "52.235.35.202", "duration": 302, "status": 404, "request": "/.env", "urlpath": "/.env", "urlquery": "", "bytes": 275, "method": "GET", "site": "120.27.220.47", "referer": "-", "useragent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" }
{ "@timestamp": "2021-12-31T14:29:05+0800", "@version": "1", "tags":["apache"], "message": "52.235.35.202 - - [31/Dec/2021:14:29:05 +0800] \"POST / HTTP/1.1\" 200 3138", "clientip": "52.235.35.202", "duration": 846, "status": 200, "request": "/index.html", "urlpath": "/index.html", "urlquery": "", "bytes": 3138, "method": "POST", "site": "120.27.220.47", "referer": "-", "useragent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" }
1. 刷新页面¶
# 10.0.8.1
2. 查看访问日志¶
root@ali:/etc/apache2# tail -f /var/log/apache2/access.log
{ "@timestamp": "2021-12-31T14:28:49+0800", "@version": "1", "tags":["apache"], "message": "52.235.35.202 - - [31/Dec/2021:14:28:49 +0800] \"GET /.env HTTP/1.1\" 404 275", "clientip": "52.235.35.202", "duration": 302, "status": 404, "request": "/.env", "urlpath": "/.env", "urlquery": "", "bytes": 275, "method": "GET", "site": "120.27.220.47", "referer": "-", "useragent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" }
{ "@timestamp": "2021-12-31T14:29:05+0800", "@version": "1", "tags":["apache"], "message": "52.235.35.202 - - [31/Dec/2021:14:29:05 +0800] \"POST / HTTP/1.1\" 200 3138", "clientip": "52.235.35.202", "duration": 846, "status": 200, "request": "/index.html", "urlpath": "/index.html", "urlquery": "", "bytes": 3138, "method": "POST", "site": "120.27.220.47", "referer": "-", "useragent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" }
# 增加了下面的日志量!
{ "@timestamp": "2021-12-31T14:36:20+0800", "@version": "1", "tags":["apache"], "message": "10.0.8.5 - - [31/Dec/2021:14:36:20 +0800] \"GET / HTTP/1.1\" 200 3138", "clientip": "10.0.8.5", "duration": 946, "status": 200, "request": "/index.html", "urlpath": "/index.html", "urlquery": "", "bytes": 3138, "method": "GET", "site": "10.0.8.1", "referer": "-", "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36" }
{ "@timestamp": "2021-12-31T14:36:21+0800", "@version": "1", "tags":["apache"], "message": "10.0.8.5 - - [31/Dec/2021:14:36:21 +0800] \"GET /icons/ubuntu-logo.png HTTP/1.1\" 200 3338", "clientip": "10.0.8.5", "duration": 314, "status": 200, "request": "/icons/ubuntu-logo.png", "urlpath": "/icons/ubuntu-logo.png", "urlquery": "", "bytes": 3338, "method": "GET", "site": "10.0.8.1", "referer": "http://10.0.8.1/", "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36" }
{ "@timestamp": "2021-12-31T14:36:21+0800", "@version": "1", "tags":["apache"], "message": "10.0.8.5 - - [31/Dec/2021:14:36:21 +0800] \"GET /favicon.ico HTTP/1.1\" 404 270", "clientip": "10.0.8.5", "duration": 221, "status": 404, "request": "/favicon.ico", "urlpath": "/favicon.ico", "urlquery": "", "bytes": 270, "method": "GET", "site": "10.0.8.1", "referer": "http://10.0.8.1/", "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36" }
二、安装filebeat¶
5. dpkg安装filebeat¶
# 10.0.8.1 # 内存从0.59->0.64G! 大概使用0.05G内存!!!
sudo -i
cd /home/admin
dpkg -i filebeat*
root@ali:/etc/apache2# sudo -i
root@ali:~# cd /home/admin
root@ali:/home/admin# dpkg -i filebeat*
Selecting previously unselected package filebeat.
(Reading database ... 116123 files and directories currently installed.)
Preparing to unpack filebeat-7.16.2-amd64.deb ...
Unpacking filebeat (7.16.2) ...
Setting up filebeat (7.16.2) ...
Processing triggers for systemd (245.4-4ubuntu3.2) ...
6.修改filebeat配置文件¶
1. 备份¶
cd /etc/filebeat
ls
cp filebeat.yml filebeat.yml.bak
root@ali:/home/admin# cd /etc/filebeat
root@ali:/etc/filebeat# ls
fields.yml filebeat.reference.yml filebeat.yml modules.d
root@ali:/etc/filebeat# cp filebeat.yml filebeat.yml.bak
2. 修改¶
vim /etc/filebeat/filebeat.yml
#1. 在inputs中
#1 修改25行,将false改为true,启用日志监控
enabled: true
#2 修改22行,类型从filestream改为log
- type: log
#3 修改29行,监控apache访问日志
- /var/log/apache2/access.log
#4 添加31和32行,支持读取json格式,注意:以下2行与paths对齐!
json.keys_under_root: true
json.overwrite_keys: true
#2. 在outputs中
#1 自定义索引名称
#1. 131行下面,定义索引名称和索引样式,自己随便起,能区分开就行!
# 这两行顶头开始写!
setup.template.name: "10.0.8.1"
setup.template.pattern: "10.0.8.1-"
#2. 添加134行,索引生命周期管理ilm功能默认开启,开启的情况下索引名称只能为filebeat-*
# 这1行顶头开始写!
setup.ilm.enabled: false
#3. 137行下面,添加刚才定义的索引
# 这一行开头2个空格!
index: "10.0.8.1-apache2-accesslog_%{+yyyy.MM.dd}"
#2 elasticsearch所在的主机
hosts: ["10.0.8.5:9200"]
1. filebeat inputs中的修改内容¶
#1.
13 # ============================== Filebeat inputs ===============================
14
15 filebeat.inputs:
16
17 # Each - is an input. Most options can be set at the input level, so
18 # you can use different inputs for various configurations.
19 # Below are the input specific configurations.
20
21 # filestream is an input for collecting log messages from files.
22 - type: log
23
24 # Change to true to enable this input configuration.
25 enabled: true
26
27 # Paths that should be crawled and fetched. Glob based paths.
28 paths:
29 - /var/log/apache2/access.log
30 #- c:\programdata\elasticsearch\logs\*
31 json.keys_under_root: true
32 json.overwrite_keys: true
33
34 # Exclude lines. A list of regular expressions to match. It drops the lines that are
35 # matching any regular expression from the list.
36 #exclude_lines: ['^DBG']
37
38 # Include lines. A list of regular expressions to match. It exports the lines that are
39 # matching any regular expression from the list.
40 #include_lines: ['^ERR', '^WARN']
41
42 # Exclude files. A list of regular expressions to match. Filebeat drops the files that
43 # are matching any regular expression from the list. By default, no files are dropped.
44 #prospector.scanner.exclude_files: ['.gz$']
45
46 # Optional additional fields. These fields can be freely picked
47 # to add additional information to the crawled log files for filtering
48 #fields:
49 # level: debug
50 # review: 1
2. 在elasticsearch output中的修改内容¶
129 # ================================== Outputs ===================================
130
131 # Configure what output to use when sending the data collected by the beat.
132
133 # ---------------------------- Elasticsearch Output ----------------------------
134 setup.template.name: "10.0.8.1"
135 setup.template.pattern: "10.0.8.1-"
136 setup.ilm.enabled: false
137 output.elasticsearch:
138 # Array of hosts to connect to.
139 hosts: ["10.0.8.5:9200"]
140 index: "10.0.8.1-apache2-accesslog_%{+yyyy.MM.dd}"
141 # Protocol - either `http` (default) or `https`.
142 #protocol: "https"
143
144 # Authentication credentials - either API key or username/password.
145 #api_key: "id:api_key"
146 #username: "elastic"
147 #password: "changeme"
7. 重启filebeat服务¶
systemctl restart filebeat
三、查看elasticsearch索引¶
8. 删除elasticsearch中的索引¶
# 10.0.8.5
#1.删除filebeat索引
curl -XDELETE 127.0.0.1:9200/filebeat*
#2.查看索引
curl 127.0.0.1:9200/_cat/indices
root@4c16g:~# curl 127.0.0.1:9200/_cat/indices
green open .geoip_databases vp77wdCxR6ersomYKR7XIg 1 0 44 0 40.8mb 40.8mb
yellow open 10.0.8.5-syslog-chupeng_2021.12.31 FTqhnt5iRx2zUwM5tKzg5A 1 1 5 0 73kb 73kb
green open .kibana_task_manager_7.16.2_001 1n0PxyQoSJKKmUs20b79qA 1 0 17 10753 1.3mb 1.3mb
green open .apm-custom-link NwEzmXWMRb-Lnnw_Y8C33Q 1 0 0 0 226b 226b
green open .kibana_7.16.2_001 LCun_7ggQiekuppoURCj8w 1 0 267 62 4.9mb 4.9mb
green open .apm-agent-configuration kQzGyZJxSSaotJkeBbpVbQ 1 0 0 0 226b 226b
green open .async-search 9RuXcU90T1OMOX-AZ6YDZA 1 0 4 0 9.3kb 9.3kb
green open .tasks rDc3gYaqTtiYoUe7TPlBig 1 0 4 0 21.4kb 21.4kb
9. 生成apache2访问日志,再次查看索引¶
# 10.0.8.1
#2.再次查看索引
curl 127.0.0.1:9200/_cat/indices
root@4c16g:~# curl 127.0.0.1:9200/_cat/indices
green open .geoip_databases vp77wdCxR6ersomYKR7XIg 1 0 44 0 40.8mb 40.8mb
yellow open 10.0.8.5-syslog-chupeng_2021.12.31 FTqhnt5iRx2zUwM5tKzg5A 1 1 5 0 73kb 73kb
green open .kibana_task_manager_7.16.2_001 1n0PxyQoSJKKmUs20b79qA 1 0 17 10753 1.3mb 1.3mb
green open .apm-custom-link NwEzmXWMRb-Lnnw_Y8C33Q 1 0 0 0 226b 226b
green open .kibana_7.16.2_001 LCun_7ggQiekuppoURCj8w 1 0 267 62 4.9mb 4.9mb
green open .apm-agent-configuration kQzGyZJxSSaotJkeBbpVbQ 1 0 0 0 226b 226b
green open .async-search 9RuXcU90T1OMOX-AZ6YDZA 1 0 4 0 9.3kb 9.3kb
yellow open 10.0.8.1-apache2-accesslog_2021.12.31 o2yQJeh0QOSTYwlf4FHjoQ 1 1 1 0 20.8kb 20.8kb #产生索引!
green open .tasks rDc3gYaqTtiYoUe7TPlBig 1 0 4 0 21.4kb 21.4kb
10. 登录kibana查看¶
# 10.0.8.5:5601
# 点击左侧列表中的discover,然后点击 ··· 进入索引管理页面
# 先将10.0.8.5的索引删除
# 然后重新添加10.0.8.1的索引
# 然后点击discover,查看日志,刷新:10.0.8.1 查看是否多产生日志!
问题:未单独显示出json数据!¶
# 查看日志内容,有问题,虽然json格式发过去了,但是没有在kibana中将json数据显示出来!
四、修改filebeat配置文件后¶
# 将日志类型从filestream改为log后,即可得到json格式日志中的关键字了!!!
最后更新:
2022-02-19 13:05:46