14. 设置filebeat将多行日志合并¶
1. multiline介绍¶
# 10.0.8.5
# multiline.pattern: 正则表达式
# multiline.negate: 默认是false,匹配pattern的行合并到上一行;true,不匹配pattern的行合并到上一行
# multiline.match: after:合并到上一行的末尾,before:合并到上一行的开头
2. 多行合并思路¶
1. 观察慢日志信息¶
tail -f /usr/local/mysql/data/web-slow.log
# Time: 2021-12-10T07:00:04.923751Z
# User@Host: root[root] @ localhost [] Id: 5
# Query_time: 6.000753 Lock_time: 0.000097 Rows_sent: 1 Rows_examined: 19999999
SET timestamp=1639119604;
select * from db1.t1 where id=18;
2. 思路¶
# 正则匹配到不以 # User@Host:开头的行,则合并到上一行的末尾
3. 效果如下¶
# User@Host: root[root] @ localhost [] Id: 5# Query_time: 6.000753 Lock_time: 0.000097 Rows_sent: 1 Rows_examined: 19999999SET timestamp=1639119604;select * from db1.t1 where id=18;#Time: 2021-12-10T07:00:04.923751Z
4. 配置文件添加¶
multiline.pattern: "^# User@Host:"
multiline.negate: true
multiline.match: after
3. 修改filebeat配置文件¶
vim /etc/filebeat/filebeat.yml
# 在慢日志监控下,添加多行合并配置!添加20-22行,注意yaml格式,开头空2格!
multiline.pattern: "^# User@Host:"
multiline.negate: true
multiline.match: after
13 # ============================== Filebeat inputs ===============================
14
15 filebeat.inputs:
16 - type: log
17 enabled: True
18 paths:
19 - /usr/local/mysql/data/web-slow.log
20 multiline.pattern: "^# User@Host:"
21 multiline.negate: true
22 multiline.match: after
23 # Each - is an input. Most options can be set at the input level, so
24 # you can use different inputs for various configurations.
25 # Below are the input specific configurations.
26
27 # filestream is an input for collecting log messages from files.
28 #- type: filestream
29
30 # Change to true to enable this input configuration.
31 # enabled: true
32
33 # Paths that should be crawled and fetched. Glob based paths.
34 #paths:
35 # - /var/log/chup.log
36 #- c:\programdata\elasticsearch\logs\*
37
38 # Exclude lines. A list of regular expressions to match. It drops the lines that are
39 # matching any regular expression from the list.
40 #exclude_lines: ['^DBG']
41
42 # Include lines. A list of regular expressions to match. It exports the lines that are
43 # matching any regular expression from the list.
44 #include_lines: ['^ERR', '^WARN','sshd']
45
46 # Exclude files. A list of regular expressions to match. Filebeat drops the files that
47 # are matching any regular expression from the list. By default, no files are dropped.
48 #prospector.scanner.exclude_files: ['.gz$']
49
50 # Optional additional fields. These fields can be freely picked
51 # to add additional information to the crawled log files for filtering
52 #fields:
53 # level: debug
54 # review: 1
4. 重启filebeat¶
systemctl restart filebeat
5. 再次生成慢日志¶
mysql -uroot -p123456
select * from db1.t1 where id=8;
6. 查看logstash接收到的内容¶
# 发现:日志从5条变成2条了!
#第一条,无用,想办法将其删除!
"message" => "# Time: 2021-12-10T07:17:17.100835Z"
#第二条,正是我们想要的内容!
"message" => "# User@Host: root[root] @ localhost [] Id: 6\n# Query_time: 5.759042 Lock_time: 0.000231 Rows_sent: 1 Rows_examined: 19999999\nSET timestamp=1639120637;\nselect * from db1.t1 where id=8;"
[WARN ] 2022-01-01 22:30:57.556 [defaultEventExecutorGroup-4-2] plain - Relying on default value of `pipeline.ecs_compatibility`, which may change in a future major release of Logstash. To avoid unexpected changes when upgrading Logstash, please explicitly declare your desired ECS Compatibility mode.
#1.第一条
{
"ecs" => {
"version" => "1.12.0"
},
"log" => {
"offset" => 877,
"file" => {
"path" => "/usr/local/mysql/data/web-slow.log"
}
},
"host" => {
"architecture" => "x86_64",
"id" => "b3856b2bce5c47ab962ede7e592b054c",
"ip" => [
[ 0] "10.0.16.15",
[ 1] "fe80::5054:ff:fe3c:125f",
[ 2] "172.17.0.1",
[ 3] "fe80::42:e8ff:fe51:6b3d",
[ 4] "10.0.8.5",
[ 5] "10.244.75.128",
[ 6] "192.168.250.1",
[ 7] "fe80::42:f0ff:fe96:2200",
[ 8] "172.18.0.1",
[ 9] "fe80::42:aeff:fe33:d359",
[10] "fe80::40d0:ff:fe10:12eb",
[11] "fe80::b82e:bdff:fe27:16c5",
[12] "fe80::c9b:5ff:fe8b:48de",
[13] "fe80::ecee:eeff:feee:eeee",
[14] "fe80::ecee:eeff:feee:eeee",
[15] "fe80::f47c:4aff:fee2:37c",
[16] "fe80::ecee:eeff:feee:eeee",
[17] "fe80::f409:3aff:fe34:31e5",
[18] "fe80::ecee:eeff:feee:eeee",
[19] "fe80::ecee:eeff:feee:eeee",
[20] "fe80::a8c6:5ff:fe0a:f9c8",
[21] "fe80::cc6b:23ff:fe9b:4ac2",
[22] "fe80::706b:1aff:fe93:c09c",
[23] "fe80::f8e0:abff:fec5:9520",
[24] "fe80::da:89ff:fe5b:2491",
[25] "fe80::642c:c8ff:fe3b:bedc",
[26] "fe80::481d:16ff:fe20:266"
],
"os" => {
"platform" => "ubuntu",
"codename" => "focal",
"kernel" => "5.4.0-90-generic",
"name" => "Ubuntu",
"type" => "linux",
"family" => "debian",
"version" => "20.04 LTS (Focal Fossa)"
},
"name" => "4c16g",
"hostname" => "4c16g",
"containerized" => false,
"mac" => [
[ 0] "52:54:00:3c:12:5f",
[ 1] "02:42:e8:51:6b:3d",
[ 2] "02:42:f0:96:22:00",
[ 3] "02:42:ae:33:d3:59",
[ 4] "42:d0:00:10:12:eb",
[ 5] "ba:2e:bd:27:16:c5",
[ 6] "0e:9b:05:8b:48:de",
[ 7] "ee:ee:ee:ee:ee:ee",
[ 8] "ee:ee:ee:ee:ee:ee",
[ 9] "f6:7c:4a:e2:03:7c",
[10] "ee:ee:ee:ee:ee:ee",
[11] "f6:09:3a:34:31:e5",
[12] "ee:ee:ee:ee:ee:ee",
[13] "ee:ee:ee:ee:ee:ee",
[14] "aa:c6:05:0a:f9:c8",
[15] "ce:6b:23:9b:4a:c2",
[16] "72:6b:1a:93:c0:9c",
[17] "fa:e0:ab:c5:95:20",
[18] "02:da:89:5b:24:91",
[19] "66:2c:c8:3b:be:dc",
[20] "4a:1d:16:20:02:66"
]
},
"input" => {
"type" => "log"
},
"@timestamp" => 2022-01-01T14:30:56.539Z,
"message" => "# Time: 2022-01-01T14:30:52.398839Z",
"tags" => [
[0] "beats_input_codec_plain_applied"
],
"@version" => "1",
"agent" => {
"version" => "7.16.2",
"id" => "edca8f85-bc52-4365-b94b-bdf17eb8209f",
"name" => "4c16g",
"hostname" => "4c16g",
"type" => "filebeat",
"ephemeral_id" => "3568ddcd-f4a5-499f-84f4-1feac879b4b9"
},
"container" => {
"id" => "web-slow.log"
}
}
#2.第二条
{
"ecs" => {
"version" => "1.12.0"
},
"log" => {
"flags" => [
[0] "multiline"
],
"offset" => 913,
"file" => {
"path" => "/usr/local/mysql/data/web-slow.log"
}
},
"host" => {
"architecture" => "x86_64",
"id" => "b3856b2bce5c47ab962ede7e592b054c",
"os" => {
"platform" => "ubuntu",
"codename" => "focal",
"kernel" => "5.4.0-90-generic",
"name" => "Ubuntu",
"type" => "linux",
"family" => "debian",
"version" => "20.04 LTS (Focal Fossa)"
},
"ip" => [
[ 0] "10.0.16.15",
[ 1] "fe80::5054:ff:fe3c:125f",
[ 2] "172.17.0.1",
[ 3] "fe80::42:e8ff:fe51:6b3d",
[ 4] "10.0.8.5",
[ 5] "10.244.75.128",
[ 6] "192.168.250.1",
[ 7] "fe80::42:f0ff:fe96:2200",
[ 8] "172.18.0.1",
[ 9] "fe80::42:aeff:fe33:d359",
[10] "fe80::40d0:ff:fe10:12eb",
[11] "fe80::b82e:bdff:fe27:16c5",
[12] "fe80::c9b:5ff:fe8b:48de",
[13] "fe80::ecee:eeff:feee:eeee",
[14] "fe80::ecee:eeff:feee:eeee",
[15] "fe80::f47c:4aff:fee2:37c",
[16] "fe80::ecee:eeff:feee:eeee",
[17] "fe80::f409:3aff:fe34:31e5",
[18] "fe80::ecee:eeff:feee:eeee",
[19] "fe80::ecee:eeff:feee:eeee",
[20] "fe80::a8c6:5ff:fe0a:f9c8",
[21] "fe80::cc6b:23ff:fe9b:4ac2",
[22] "fe80::706b:1aff:fe93:c09c",
[23] "fe80::f8e0:abff:fec5:9520",
[24] "fe80::da:89ff:fe5b:2491",
[25] "fe80::642c:c8ff:fe3b:bedc",
[26] "fe80::481d:16ff:fe20:266"
],
"name" => "4c16g",
"hostname" => "4c16g",
"containerized" => false,
"mac" => [
[ 0] "52:54:00:3c:12:5f",
[ 1] "02:42:e8:51:6b:3d",
[ 2] "02:42:f0:96:22:00",
[ 3] "02:42:ae:33:d3:59",
[ 4] "42:d0:00:10:12:eb",
[ 5] "ba:2e:bd:27:16:c5",
[ 6] "0e:9b:05:8b:48:de",
[ 7] "ee:ee:ee:ee:ee:ee",
[ 8] "ee:ee:ee:ee:ee:ee",
[ 9] "f6:7c:4a:e2:03:7c",
[10] "ee:ee:ee:ee:ee:ee",
[11] "f6:09:3a:34:31:e5",
[12] "ee:ee:ee:ee:ee:ee",
[13] "ee:ee:ee:ee:ee:ee",
[14] "aa:c6:05:0a:f9:c8",
[15] "ce:6b:23:9b:4a:c2",
[16] "72:6b:1a:93:c0:9c",
[17] "fa:e0:ab:c5:95:20",
[18] "02:da:89:5b:24:91",
[19] "66:2c:c8:3b:be:dc",
[20] "4a:1d:16:20:02:66"
]
},
"@timestamp" => 2022-01-01T14:30:56.539Z,
"message" => "# User@Host: root[root] @ localhost [] Id: 5\n# Query_time: 8.120293 Lock_time: 0.000173 Rows_sent: 1 Rows_examined: 19999999\nSET timestamp=1641047452;\nselect * from db1.t1 where id=8;",
"input" => {
"type" => "log"
},
"tags" => [
[0] "beats_input_codec_plain_applied"
],
"@version" => "1",
"agent" => {
"ephemeral_id" => "3568ddcd-f4a5-499f-84f4-1feac879b4b9",
"id" => "edca8f85-bc52-4365-b94b-bdf17eb8209f",
"hostname" => "4c16g",
"name" => "4c16g",
"type" => "filebeat",
"version" => "7.16.2"
},
"container" => {
"id" => "web-slow.log"
}
}
最后更新:
2022-02-19 13:05:46