一、安装nginx¶
# 10.0.8.4
# 注意问题：修改filebeat配置文件时，日志文件类型要修改为 - type：log ！！！
1. 安装nginx¶
# 安装nginx！
apt install nginx -y
2. 修改nginx配置文件¶
1. 查看配置文件¶
ls /etc/nginx/
root@vpn:~# ls /etc/nginx/
conf.d fastcgi_params koi-win modules-available nginx.conf scgi_params sites-enabled uwsgi_params
fastcgi.conf koi-utf mime.types modules-enabled proxy_params sites-available snippets win-utf
2. 修改配置文件¶
1. 备份¶
cd /etc/nginx/
cp nginx.conf nginx.conf.bak
2.添加自定义json日志格式！¶
vim nginx.conf
#1.新加自定义json日志格式！
#在http大括号中加40-49行
log_format main_json '{"@timestamp":"$time_local",'
'"N_client_ip": "$remote_addr",'
'"N_request": "$request",'
'"N_request_time": "$request_time",'
'"N_status": "$status",'
'"N_bytes": "$body_bytes_sent",'
'"N_user_agent": "$http_user_agent",'
'"N_x_forwarded": "$http_x_forwarded_for",'
'"N_referer": "$http_referer"'
'}';
#2.修改51行，访问日志为json格式！
#3.注释第72行，site-enabled中的配置文件
37 ##
38 # Logging Settings
39 ##
40 log_format main_json '{"@timestamp":"$time_local",'
41 '"N_client_ip": "$remote_addr",'
42 '"N_request": "$request",'
43 '"N_request_time": "$request_time",'
44 '"N_status": "$status",'
45 '"N_bytes": "$body_bytes_sent",'
46 '"N_user_agent": "$http_user_agent",'
47 '"N_x_forwarded": "$http_x_forwarded_for",'
48 '"N_referer": "$http_referer"'
49 '}';
50
51 access_log /var/log/nginx/access.log main_json;
52 error_log /var/log/nginx/error.log;
53
。。。
71 include /etc/nginx/conf.d/*.conf;
72 #include /etc/nginx/sites-enabled/*;
73 }
3. 创建80端口配置文件¶
cd conf.d
vim 80.port
server {
listen 80;
listen [::]:80;
root /var/www/html;
index index.html;
location / {
try_files $uri $uri/ =404;
}
}
4. 创建html¶
cd /var/www/html
vim index.html
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
3. 启动nginx¶
nginx -t
systemctl start nginx
netstat -tunlp|grep 80
4. 监控访问日志¶
tail -f /var/log/nginx/access.log
# 发现日志确实已经变为json格式了！！！
root@ali:/etc/nginx# tail -f /var/log/nginx/access.log
{ "@timestamp": "2021-12-31T14:28:49+0800", "@version": "1", "tags":["apache"], "message": "52.235.35.202 - - [31/Dec/2021:14:28:49 +0800] \"GET /.env HTTP/1.1\" 404 275", "clientip": "52.235.35.202", "duration": 302, "status": 404, "request": "/.env", "urlpath": "/.env", "urlquery": "", "bytes": 275, "method": "GET", "site": "120.27.220.47", "referer": "-", "useragent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" }
{ "@timestamp": "2021-12-31T14:29:05+0800", "@version": "1", "tags":["apache"], "message": "52.235.35.202 - - [31/Dec/2021:14:29:05 +0800] \"POST / HTTP/1.1\" 200 3138", "clientip": "52.235.35.202", "duration": 846, "status": 200, "request": "/index.html", "urlpath": "/index.html", "urlquery": "", "bytes": 3138, "method": "POST", "site": "120.27.220.47", "referer": "-", "useragent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" }
1. 刷新页面¶
# 10.0.8.4
2. 查看访问日志¶
# tail -f /var/log/nginx/access.log
{ "@timestamp": "2021-12-31T14:28:49+0800", "@version": "1", "tags":["apache"], "message": "52.235.35.202 - - [31/Dec/2021:14:28:49 +0800] \"GET /.env HTTP/1.1\" 404 275", "clientip": "52.235.35.202", "duration": 302, "status": 404, "request": "/.env", "urlpath": "/.env", "urlquery": "", "bytes": 275, "method": "GET", "site": "120.27.220.47", "referer": "-", "useragent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" }
{ "@timestamp": "2021-12-31T14:29:05+0800", "@version": "1", "tags":["apache"], "message": "52.235.35.202 - - [31/Dec/2021:14:29:05 +0800] \"POST / HTTP/1.1\" 200 3138", "clientip": "52.235.35.202", "duration": 846, "status": 200, "request": "/index.html", "urlpath": "/index.html", "urlquery": "", "bytes": 3138, "method": "POST", "site": "120.27.220.47", "referer": "-", "useragent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36" }
# 增加了下面的日志量！
{ "@timestamp": "2021-12-31T14:36:20+0800", "@version": "1", "tags":["apache"], "message": "10.0.8.5 - - [31/Dec/2021:14:36:20 +0800] \"GET / HTTP/1.1\" 200 3138", "clientip": "10.0.8.5", "duration": 946, "status": 200, "request": "/index.html", "urlpath": "/index.html", "urlquery": "", "bytes": 3138, "method": "GET", "site": "10.0.8.4", "referer": "-", "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36" }
{ "@timestamp": "2021-12-31T14:36:21+0800", "@version": "1", "tags":["apache"], "message": "10.0.8.5 - - [31/Dec/2021:14:36:21 +0800] \"GET /icons/ubuntu-logo.png HTTP/1.1\" 200 3338", "clientip": "10.0.8.5", "duration": 314, "status": 200, "request": "/icons/ubuntu-logo.png", "urlpath": "/icons/ubuntu-logo.png", "urlquery": "", "bytes": 3338, "method": "GET", "site": "10.0.8.4", "referer": "http://10.0.8.4/", "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36" }
{ "@timestamp": "2021-12-31T14:36:21+0800", "@version": "1", "tags":["apache"], "message": "10.0.8.5 - - [31/Dec/2021:14:36:21 +0800] \"GET /favicon.ico HTTP/1.1\" 404 270", "clientip": "10.0.8.5", "duration": 221, "status": 404, "request": "/favicon.ico", "urlpath": "/favicon.ico", "urlquery": "", "bytes": 270, "method": "GET", "site": "10.0.8.4", "referer": "http://10.0.8.4/", "useragent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36" }
二、安装filebeat¶
5. dpkg安装filebeat¶
# 10.0.8.4 # 内存从0.59->0.64G! 大概使用0.05G内存！！！
sudo -i
cd /home/ubuntu
ls
dpkg -i filebeat*
6.修改filebeat配置文件¶
1. 备份¶
cd /etc/filebeat
ls
cp filebeat.yml filebeat.yml.bak
2. 修改¶
vim /etc/filebeat/filebeat.yml
#1. 在inputs中
#1 修改25行，将false改为true，启用日志监控
enabled: true
#2 修改29行，监控apache访问日志
- /var/log/nginx/access.log
#3 添加31和32行，支持读取json格式，注意：以下2行与paths对齐！
json.keys_under_root: true
json.overwrite_keys: true
#2. 在outputs中
#1 自定义索引名称
#1. 131行下面，定义索引名称和索引样式，自己随便起，能区分开就行！
# 这两行顶头开始写！
setup.template.name: "10.0.8.4"
setup.template.pattern: "10.0.8.4-"
#2. 添加134行，索引生命周期管理ilm功能默认开启，开启的情况下索引名称只能为filebeat-*
# 这1行顶头开始写！
setup.ilm.enabled: false
#3. 137行下面，添加刚才定义的索引
# 这一行开头2个空格！
index: "10.0.8.4-nginx-accesslog_%{+yyyy.MM.dd}"
#2 elasticsearch所在的主机
hosts: ["10.0.8.5:9200"]
1. filebeat inputs中的修改内容¶
#1.
13 # ============================== Filebeat inputs ===============================
14
15 filebeat.inputs:
16
17 # Each - is an input. Most options can be set at the input level, so
18 # you can use different inputs for various configurations.
19 # Below are the input specific configurations.
20
21 # filestream is an input for collecting log messages from files.
22 - type: filestream
23
24 # Change to true to enable this input configuration.
25 enabled: true
26
27 # Paths that should be crawled and fetched. Glob based paths.
28 paths:
29 - /var/log/nginx/access.log
30 #- c:\programdata\elasticsearch\logs\*
31 json.keys_under_root: true
32 json.overwrite_keys: true
33
34 # Exclude lines. A list of regular expressions to match. It drops the lines that are
35 # matching any regular expression from the list.
36 #exclude_lines: ['^DBG']
37
38 # Include lines. A list of regular expressions to match. It exports the lines that are
39 # matching any regular expression from the list.
40 #include_lines: ['^ERR', '^WARN']
41
42 # Exclude files. A list of regular expressions to match. Filebeat drops the files that
43 # are matching any regular expression from the list. By default, no files are dropped.
44 #prospector.scanner.exclude_files: ['.gz$']
45
46 # Optional additional fields. These fields can be freely picked
47 # to add additional information to the crawled log files for filtering
48 #fields:
49 # level: debug
50 # review: 1
2. 在elasticsearch output中的修改内容¶
129 # ================================== Outputs ===================================
130
131 # Configure what output to use when sending the data collected by the beat.
132
133 # ---------------------------- Elasticsearch Output ----------------------------
134 setup.template.name: "10.0.8.4"
135 setup.template.pattern: "10.0.8.4-"
136 setup.ilm.enabled: false
137 output.elasticsearch:
138 # Array of hosts to connect to.
139 hosts: ["10.0.8.5:9200"]
140 index: "10.0.8.4-nginx-accesslog_%{+yyyy.MM.dd}"
141 # Protocol - either `http` (default) or `https`.
142 #protocol: "https"
143
144 # Authentication credentials - either API key or username/password.
145 #api_key: "id:api_key"
146 #username: "elastic"
147 #password: "changeme"
7. 重启filebeat服务¶
systemctl restart filebeat
三、查看elasticsearch索引¶
8. 删除elasticsearch中的索引¶
# 10.0.8.5
#1.删除filebeat索引
curl -XDELETE 127.0.0.1:9200/filebeat*
#2.查看索引
curl 127.0.0.1:9200/_cat/indices
9. 生成nginx访问日志，再次查看索引¶
# 10.0.8.4
#2.再次查看索引
curl 127.0.0.1:9200/_cat/indices
root@4c16g:~# curl 127.0.0.1:9200/_cat/indices
yellow open 10.0.8.5-syslog-chupeng_2021.12.31 FTqhnt5iRx2zUwM5tKzg5A 1 1 208 0 318.4kb 318.4kb
green open .geoip_databases vp77wdCxR6ersomYKR7XIg 1 0 44 0 40.8mb 40.8mb
yellow open 10.0.8.4-nginx-accesslog_2021.12.31 Pm08TYbQSHG62bF_g3i2HQ 1 1 25 0 48.9kb 48.9kb #这里出现了
green open .apm-custom-link NwEzmXWMRb-Lnnw_Y8C33Q 1 0 0 0 226b 226b
green open .kibana_task_manager_7.16.2_001 1n0PxyQoSJKKmUs20b79qA 1 0 17 30976 3.3mb 3.3mb
green open .kibana_7.16.2_001 LCun_7ggQiekuppoURCj8w 1 0 287 1 2.3mb 2.3mb
green open .apm-agent-configuration kQzGyZJxSSaotJkeBbpVbQ 1 0 0 0 226b 226b
green open .async-search 9RuXcU90T1OMOX-AZ6YDZA 1 0 0 0 252b 252b
yellow open 10.0.8.1-apache2-accesslog_2021.12.31 BHOuCRLyRHu-rHIzmnn11g 1 1 19 0 68.3kb 68.3kb
green open .tasks rDc3gYaqTtiYoUe7TPlBig 1 0 4 0 21.4kb 21.4kb
10. 登录kibana查看¶
# 10.0.8.5:5601
# 点击左侧列表中的discover，然后点击 ··· 进入索引管理页面
# 先将其他的索引删除
# 然后添加索引
# 发现没有将json格式解析出来！！！
四、疑问¶
# 是不是7.16.2中以下2个参数又变了？？？
json.keys_under_root: true # 使用root权限
json.overwrite_keys: true # 使json日志替换自带的关键字
五、修改filebeat配置文件¶
# 查找官网资料发现，貌似是因为配置文件中的类型写错了，重新修改后测试
1. 修改filebeat配置文件¶
# 10.0.8.4
cd /etc/filebeat
vim filebeat.yml
# 修改22行，将filestream改为log
21 # filestream is an input for collecting log messages from files.
22 - type: log
2. 停止filebeat¶
systemctl stop filebeat
3. 删除索引¶
# 10.0.8.5
#1.删除filebeat索引
curl -XDELETE 127.0.0.1:9200/10.0.8.4*
#2.查看索引
curl 127.0.0.1:9200/_cat/indices
4. 启动filebeat¶
systemctl start filebeat
5. 查看索引¶
# 10.0.8.5
curl 127.0.0.1:9200/_cat/indices
6. kibana上查看¶
# 10.0.8.5:9200
# 发现出现了json中的关键字！！！
最后更新:
2022-02-19 13:05:46