跳转至

一、安装apache2

# 10.0.8.1

1. 安装apache2

# 安装apache2!
apt install apache2 -y

2. 修改apache2配置文件

1. 查看配置文件

ls /etc/apache2/
root@ali:~# ls /etc/apache2/
apache2.conf  conf-available  conf-enabled  envvars  magic  mods-available  mods-enabled  ports.conf  sites-available  sites-enabled

2. 修改配置文件

1. 添加自定义json日志格式!

# 添加自定义json日志格式
cd /etc/apache2/
vim apache2.conf
#1.在216行后面,新加自定义json日志格式!
LogFormat "{ \
\"@timestamp\": \"%{%Y-%m-%dT%H:%M:%S%z}t\", \
\"@version\": \"1\", \
\"tags\":[\"apache\"], \
\"message\": \"%h %l %u %t \\\"%r\\\" %>s %b\", \
\"clientip\": \"%a\", \
\"duration\": %D, \
\"status\": %>s, \
\"request\": \"%U%q\", \
\"urlpath\": \"%U\", \
\"urlquery\": \"%q\", \
\"bytes\": %B, \
\"method\": \"%m\", \
\"site\": \"%{Host}i\", \
\"referer\": \"%{Referer}i\", \
\"useragent\": \"%{User-agent}i\" \
}" apache_json
201 # The following directives define some format nicknames for use with
202 # a CustomLog directive.
203 #
204 # These deviate from the Common Log Format definitions in that they use %O
205 # (the actual bytes sent including headers) instead of %b (the size of the
206 # requested file), because the latter makes it impossible to detect partial
207 # requests.
208 #
209 # Note that the use of %{X-Forwarded-For}i instead of %h is not recommended.
210 # Use mod_remoteip instead.
211 #
212 LogFormat "%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" vhost_combined
213 LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" combined
214 LogFormat "%h %l %u %t \"%r\" %>s %O" common
215 LogFormat "%{Referer}i -> %U" referer
216 LogFormat "%{User-agent}i" agent
217
218 LogFormat "{ \
219 \"@timestamp\": \"%{%Y-%m-%dT%H:%M:%S%z}t\", \
220 \"@version\": \"1\", \
221 \"tags\":[\"apache\"], \
222 \"message\": \"%h %l %u %t \\\"%r\\\" %>s %b\", \
223 \"clientip\": \"%a\", \
224 \"duration\": %D, \
225 \"status\": %>s, \
226 \"request\": \"%U%q\", \
227 \"urlpath\": \"%U\", \
228 \"urlquery\": \"%q\", \
229 \"bytes\": %B, \
230 \"method\": \"%m\", \
231 \"site\": \"%{Host}i\", \
232 \"referer\": \"%{Referer}i\", \
233 \"useragent\": \"%{User-agent}i\" \
234 }" apache_json
235

2. 访问日志采用json格式

root@ali:/etc/apache2# cd sites-enabled/
root@ali:/etc/apache2/sites-enabled# ls
000-default.conf
root@ali:/etc/apache2/sites-enabled# vim 000-default.conf

# 修改21行,将combined改为apache_json,使用json格式记录访问日志!
  1 <VirtualHost *:80>
  2         # The ServerName directive sets the request scheme, hostname and port that
  3         # the server uses to identify itself. This is used when creating
  4         # redirection URLs. In the context of virtual hosts, the ServerName
  5         # specifies what hostname must appear in the request's Host: header to
  6         # match this virtual host. For the default virtual host (this file) this
  7         # value is not decisive as it is used as a last resort host regardless.
  8         # However, you must set it for any further virtual host explicitly.
  9         #ServerName www.example.com
 10
 11         ServerAdmin webmaster@localhost
 12         DocumentRoot /var/www/html
 13
 14         # Available loglevels: trace8, ..., trace1, debug, info, notice, warn,
 15         # error, crit, alert, emerg.
 16         # It is also possible to configure the loglevel for particular
 17         # modules, e.g.
 18         #LogLevel info ssl:warn
 19
 20         ErrorLog ${APACHE_LOG_DIR}/error.log
 21         CustomLog ${APACHE_LOG_DIR}/access.log apache_json
 22
 23         # For most configuration files from conf-available/, which are
 24         # enabled or disabled at a global level, it is possible to
 25         # include a line for only one particular virtual host. For example the
 26         # following line enables the CGI configuration for this host only
 27         # after it has been globally disabled with "a2disconf".
 28         #Include conf-available/serve-cgi-bin.conf
 29 </VirtualHost>
 30
 31 # vim: syntax=apache ts=4 sw=4 sts=4 sr noet

3. 查看访问日志文件路径

cat envvars -n

# 找到:APACHE_LOG_DIR=/var/log/apache2$SUFFIX ,最终找到路径如下!
/var/log/apache2/access.log
root@ali:/etc/apache2# cat envvars -n
     1  # envvars - default environment variables for apache2ctl
     2
     3  # this won't be correct after changing uid
     4  unset HOME
     5
     6  # for supporting multiple apache2 instances
     7  if [ "${APACHE_CONFDIR##/etc/apache2-}" != "${APACHE_CONFDIR}" ] ; then
     8          SUFFIX="-${APACHE_CONFDIR##/etc/apache2-}"
     9  else
    10          SUFFIX=
    11  fi
    12
    13  # Since there is no sane way to get the parsed apache2 config in scripts, some
    14  # settings are defined via environment variables and then used in apache2ctl,
    15  # /etc/init.d/apache2, /etc/logrotate.d/apache2, etc.
    16  export APACHE_RUN_USER=www-data
    17  export APACHE_RUN_GROUP=www-data
    18  # temporary state file location. This might be changed to /run in Wheezy+1
    19  export APACHE_PID_FILE=/var/run/apache2$SUFFIX/apache2.pid
    20  export APACHE_RUN_DIR=/var/run/apache2$SUFFIX
    21  export APACHE_LOCK_DIR=/var/lock/apache2$SUFFIX
    22  # Only /var/log/apache2 is handled by /etc/logrotate.d/apache2.
    23  export APACHE_LOG_DIR=/var/log/apache2$SUFFIX
    24
    25  ## The locale used by some modules like mod_dav
    26  export LANG=C
    27  ## Uncomment the following line to use the system default locale instead:
    28  #. /etc/default/locale
    29
    30  export LANG
    31
    32  ## The command to get the status for 'apache2ctl status'.
    33  ## Some packages providing 'www-browser' need '--dump' instead of '-dump'.
    34  #export APACHE_LYNX='www-browser -dump'
    35
    36  ## If you need a higher file descriptor limit, uncomment and adjust the
    37  ## following line (default is 8192):
    38  #APACHE_ULIMIT_MAX_FILES='ulimit -n 65536'
    39
    40  ## If you would like to pass arguments to the web server, add them below
    41  ## to the APACHE_ARGUMENTS environment.
    42  #export APACHE_ARGUMENTS=''
    43
    44  ## Enable the debug mode for maintainer scripts.
    45  ## This will produce a verbose output on package installations of web server modules and web application
    46  ## installations which interact with Apache
    47  #export APACHE2_MAINTSCRIPT_DEBUG=1
root@ali:/etc/apache2# ls /var/log/apache2/
access.log  error.log  other_vhosts_access.log

补充:各个变量代表的含义

# %a: 远程IP地址  
# %A: 本地IP地址  
# %B: 已发送的字节数,不包含HTTP头  
`# %b: CLF格式的已发送字节数量,不包含HTTP头。例如当没有发送数据时,写入‘-’而不是0。  
# %{FOOBAR}e: 环境变量FOOBAR的内容  
# %f: 文件名字  
`# %h: 远程主机  
# %H 请求的协议  
# %{Foobar}i: Foobar的内容,发送给服务器的请求的标头行。  
`# %l: 远程登录名字(来自identd,如提供的话)  
# %m: 请求的方法  
# %{Foobar}n: 来自另外一个模块的注解“Foobar”的内容  
# %{Foobar}o: Foobar的内容,应答的标头行  
# %p: 服务器响应请求时使用的端口  
# %P: 响应请求的子进程ID。  
`# %q: 查询字符串(如果存在查询字符串,则包含“?”后面的部分;否则,它是一个空字符串。)  
`# %r: 请求的第一行  
# %s: 状态。对于进行内部重定向的请求,这是指*原来*请求的状态。如果用%...>s,则是指后来的请求。  
`# %t: 以公共日志时间格式表示的时间(或称为标准英文格式)  
# %{format}t: 以指定格式format表示的时间  
# %T: 为响应请求而耗费的时间,以秒计  
`# %u: 远程用户(来自auth;如果返回状态(%s)是401则可能是伪造的)  
`# %U: 用户所请求的URL路径  
# %v: 响应请求的服务器的ServerName  
# %V: 依照UseCanonicalName设置得到的服务器名字 

3. 启动apache2

systemctl start apache2

netstat -tunlp|grep 80

4. 监控访问日志

tail -f /var/log/apache2/access.log
# 发现日志确实已经变为json格式了!!!
root@ali:/etc/apache2# tail -f /var/log/apache2/access.log
{ "@timestamp": "2021-12-31T14:28:49+0800", "@version": "1", "tags":["apache"], "message": "52.235.35.202 - - [31/Dec/2021:14:28:49 +0800] \"GET /.env HTTP/1.1\" 404 275", "clientip": "52.235.35.202", "duration": 302, "status": 404, "request": "/.env", "urlpath": "/.env", "urlquery": "", "bytes": 275, "method": "GET", "site": "120.27.220.47", "referer": "-", "useragent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" }
{ "@timestamp": "2021-12-31T14:29:05+0800", "@version": "1", "tags":["apache"], "message": "52.235.35.202 - - [31/Dec/2021:14:29:05 +0800] \"POST / HTTP/1.1\" 200 3138", "clientip": "52.235.35.202", "duration": 846, "status": 200, "request": "/index.html", "urlpath": "/index.html", "urlquery": "", "bytes": 3138, "method": "POST", "site": "120.27.220.47", "referer": "-", "useragent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" }

1. 刷新页面

# 10.0.8.1

2. 查看访问日志

root@ali:/etc/apache2# tail -f /var/log/apache2/access.log
{ "@timestamp": "2021-12-31T14:28:49+0800", "@version": "1", "tags":["apache"], "message": "52.235.35.202 - - [31/Dec/2021:14:28:49 +0800] \"GET /.env HTTP/1.1\" 404 275", "clientip": "52.235.35.202", "duration": 302, "status": 404, "request": "/.env", "urlpath": "/.env", "urlquery": "", "bytes": 275, "method": "GET", "site": "120.27.220.47", "referer": "-", "useragent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" }
{ "@timestamp": "2021-12-31T14:29:05+0800", "@version": "1", "tags":["apache"], "message": "52.235.35.202 - - [31/Dec/2021:14:29:05 +0800] \"POST / HTTP/1.1\" 200 3138", "clientip": "52.235.35.202", "duration": 846, "status": 200, "request": "/index.html", "urlpath": "/index.html", "urlquery": "", "bytes": 3138, "method": "POST", "site": "120.27.220.47", "referer": "-", "useragent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" }

# 增加了下面的日志量!
{ "@timestamp": "2021-12-31T14:36:20+0800", "@version": "1", "tags":["apache"], "message": "10.0.8.5 - - [31/Dec/2021:14:36:20 +0800] \"GET / HTTP/1.1\" 200 3138", "clientip": "10.0.8.5", "duration": 946, "status": 200, "request": "/index.html", "urlpath": "/index.html", "urlquery": "", "bytes": 3138, "method": "GET", "site": "10.0.8.1", "referer": "-", "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36" }
{ "@timestamp": "2021-12-31T14:36:21+0800", "@version": "1", "tags":["apache"], "message": "10.0.8.5 - - [31/Dec/2021:14:36:21 +0800] \"GET /icons/ubuntu-logo.png HTTP/1.1\" 200 3338", "clientip": "10.0.8.5", "duration": 314, "status": 200, "request": "/icons/ubuntu-logo.png", "urlpath": "/icons/ubuntu-logo.png", "urlquery": "", "bytes": 3338, "method": "GET", "site": "10.0.8.1", "referer": "http://10.0.8.1/", "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36" }
{ "@timestamp": "2021-12-31T14:36:21+0800", "@version": "1", "tags":["apache"], "message": "10.0.8.5 - - [31/Dec/2021:14:36:21 +0800] \"GET /favicon.ico HTTP/1.1\" 404 270", "clientip": "10.0.8.5", "duration": 221, "status": 404, "request": "/favicon.ico", "urlpath": "/favicon.ico", "urlquery": "", "bytes": 270, "method": "GET", "site": "10.0.8.1", "referer": "http://10.0.8.1/", "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36" }

二、安装filebeat

5. dpkg安装filebeat

# 10.0.8.1  # 内存从0.59->0.64G! 大概使用0.05G内存!!!
sudo -i
cd /home/admin
dpkg -i filebeat*
root@ali:/etc/apache2# sudo -i
root@ali:~# cd /home/admin
root@ali:/home/admin# dpkg -i filebeat*
Selecting previously unselected package filebeat.
(Reading database ... 116123 files and directories currently installed.)
Preparing to unpack filebeat-7.16.2-amd64.deb ...
Unpacking filebeat (7.16.2) ...
Setting up filebeat (7.16.2) ...
Processing triggers for systemd (245.4-4ubuntu3.2) ...

6.修改filebeat配置文件

1. 备份

cd /etc/filebeat
ls
cp filebeat.yml filebeat.yml.bak
root@ali:/home/admin# cd /etc/filebeat
root@ali:/etc/filebeat# ls
fields.yml  filebeat.reference.yml  filebeat.yml  modules.d
root@ali:/etc/filebeat# cp filebeat.yml filebeat.yml.bak

2. 修改

vim /etc/filebeat/filebeat.yml
#1. 在inputs中
    #1 修改25行,将false改为true,启用日志监控
    enabled: true
    #2 修改22行,类型从filestream改为log
    - type: log
    #3 修改29行,监控apache访问日志
    - /var/log/apache2/access.log
    #4 添加31和32行,支持读取json格式,注意:以下2行与paths对齐!
    json.keys_under_root: true
    json.overwrite_keys: true

#2. 在outputs中
    #1 自定义索引名称
        #1. 131行下面,定义索引名称和索引样式,自己随便起,能区分开就行!
        # 这两行顶头开始写!
        setup.template.name: "10.0.8.1"
        setup.template.pattern: "10.0.8.1-"

        #2. 添加134行,索引生命周期管理ilm功能默认开启,开启的情况下索引名称只能为filebeat-*
        # 这1行顶头开始写!
        setup.ilm.enabled: false

        #3. 137行下面,添加刚才定义的索引
        # 这一行开头2个空格!
          index: "10.0.8.1-apache2-accesslog_%{+yyyy.MM.dd}"
    #2 elasticsearch所在的主机
        hosts: ["10.0.8.5:9200"]
1. filebeat inputs中的修改内容
 #1.
 13 # ============================== Filebeat inputs ===============================
 14
 15 filebeat.inputs:
 16
 17 # Each - is an input. Most options can be set at the input level, so
 18 # you can use different inputs for various configurations.
 19 # Below are the input specific configurations.
 20
 21 # filestream is an input for collecting log messages from files.
 22 - type: log
 23
 24   # Change to true to enable this input configuration.
 25   enabled: true
 26
 27   # Paths that should be crawled and fetched. Glob based paths.
 28   paths:
 29     - /var/log/apache2/access.log
 30     #- c:\programdata\elasticsearch\logs\*
 31   json.keys_under_root: true
 32   json.overwrite_keys: true
 33
 34   # Exclude lines. A list of regular expressions to match. It drops the lines that are
 35   # matching any regular expression from the list.
 36   #exclude_lines: ['^DBG']
 37
 38   # Include lines. A list of regular expressions to match. It exports the lines that are
 39   # matching any regular expression from the list.
 40   #include_lines: ['^ERR', '^WARN']
 41
 42   # Exclude files. A list of regular expressions to match. Filebeat drops the files that
 43   # are matching any regular expression from the list. By default, no files are dropped.
 44   #prospector.scanner.exclude_files: ['.gz$']
 45
 46   # Optional additional fields. These fields can be freely picked
 47   # to add additional information to the crawled log files for filtering
 48   #fields:
 49   #  level: debug
 50   #  review: 1
2. 在elasticsearch output中的修改内容
129 # ================================== Outputs ===================================
130
131 # Configure what output to use when sending the data collected by the beat.
132
133 # ---------------------------- Elasticsearch Output ----------------------------
134 setup.template.name: "10.0.8.1"
135 setup.template.pattern: "10.0.8.1-"
136 setup.ilm.enabled: false
137 output.elasticsearch:
138   # Array of hosts to connect to.
139   hosts: ["10.0.8.5:9200"]
140   index: "10.0.8.1-apache2-accesslog_%{+yyyy.MM.dd}"
141   # Protocol - either `http` (default) or `https`.
142   #protocol: "https"
143
144   # Authentication credentials - either API key or username/password.
145   #api_key: "id:api_key"
146   #username: "elastic"
147   #password: "changeme"

7. 重启filebeat服务

systemctl restart filebeat

三、查看elasticsearch索引

8. 删除elasticsearch中的索引

# 10.0.8.5
#1.删除filebeat索引
curl -XDELETE 127.0.0.1:9200/filebeat*

#2.查看索引
curl 127.0.0.1:9200/_cat/indices
root@4c16g:~# curl 127.0.0.1:9200/_cat/indices
green  open .geoip_databases                      vp77wdCxR6ersomYKR7XIg 1 0  44     0 40.8mb 40.8mb
yellow open 10.0.8.5-syslog-chupeng_2021.12.31    FTqhnt5iRx2zUwM5tKzg5A 1 1   5     0   73kb   73kb
green  open .kibana_task_manager_7.16.2_001       1n0PxyQoSJKKmUs20b79qA 1 0  17 10753  1.3mb  1.3mb
green  open .apm-custom-link                      NwEzmXWMRb-Lnnw_Y8C33Q 1 0   0     0   226b   226b
green  open .kibana_7.16.2_001                    LCun_7ggQiekuppoURCj8w 1 0 267    62  4.9mb  4.9mb
green  open .apm-agent-configuration              kQzGyZJxSSaotJkeBbpVbQ 1 0   0     0   226b   226b
green  open .async-search                         9RuXcU90T1OMOX-AZ6YDZA 1 0   4     0  9.3kb  9.3kb
green  open .tasks                                rDc3gYaqTtiYoUe7TPlBig 1 0   4     0 21.4kb 21.4kb

9. 生成apache2访问日志,再次查看索引

# 10.0.8.1

#2.再次查看索引
curl 127.0.0.1:9200/_cat/indices
root@4c16g:~# curl 127.0.0.1:9200/_cat/indices
green  open .geoip_databases                      vp77wdCxR6ersomYKR7XIg 1 0  44     0 40.8mb 40.8mb
yellow open 10.0.8.5-syslog-chupeng_2021.12.31    FTqhnt5iRx2zUwM5tKzg5A 1 1   5     0   73kb   73kb
green  open .kibana_task_manager_7.16.2_001       1n0PxyQoSJKKmUs20b79qA 1 0  17 10753  1.3mb  1.3mb
green  open .apm-custom-link                      NwEzmXWMRb-Lnnw_Y8C33Q 1 0   0     0   226b   226b
green  open .kibana_7.16.2_001                    LCun_7ggQiekuppoURCj8w 1 0 267    62  4.9mb  4.9mb
green  open .apm-agent-configuration              kQzGyZJxSSaotJkeBbpVbQ 1 0   0     0   226b   226b
green  open .async-search                         9RuXcU90T1OMOX-AZ6YDZA 1 0   4     0  9.3kb  9.3kb
yellow open 10.0.8.1-apache2-accesslog_2021.12.31 o2yQJeh0QOSTYwlf4FHjoQ 1 1   1     0 20.8kb 20.8kb   #产生索引!
green  open .tasks                                rDc3gYaqTtiYoUe7TPlBig 1 0   4     0 21.4kb 21.4kb

10. 登录kibana查看

# 10.0.8.5:5601
# 点击左侧列表中的discover,然后点击 ··· 进入索引管理页面
# 先将10.0.8.5的索引删除

img_13.png

# 然后重新添加10.0.8.1的索引

img_14.png

# 然后点击discover,查看日志,刷新:10.0.8.1 查看是否多产生日志!

img_15.png

img_16.png

问题:未单独显示出json数据!

# 查看日志内容,有问题,虽然json格式发过去了,但是没有在kibana中将json数据显示出来!

四、修改filebeat配置文件后

# 将日志类型从filestream改为log后,即可得到json格式日志中的关键字了!!!

img_17.png


最后更新: 2022-02-19 13:05:46