跳转至

14. 设置filebeat将多行日志合并

1. multiline介绍

# 10.0.8.5

# multiline.pattern: 正则表达式
# multiline.negate: 默认是false,匹配pattern的行合并到上一行;true,不匹配pattern的行合并到上一行
# multiline.match: after:合并到上一行的末尾,before:合并到上一行的开头 

2. 多行合并思路

1. 观察慢日志信息

tail -f /usr/local/mysql/data/web-slow.log
# Time: 2021-12-10T07:00:04.923751Z
# User@Host: root[root] @ localhost []  Id:     5
# Query_time: 6.000753  Lock_time: 0.000097 Rows_sent: 1  Rows_examined: 19999999
SET timestamp=1639119604;
select * from db1.t1 where id=18;

2. 思路

# 正则匹配到不以 # User@Host:开头的行,则合并到上一行的末尾

3. 效果如下

# User@Host: root[root] @ localhost []  Id:     5# Query_time: 6.000753  Lock_time: 0.000097 Rows_sent: 1  Rows_examined: 19999999SET timestamp=1639119604;select * from db1.t1 where id=18;#Time: 2021-12-10T07:00:04.923751Z

4. 配置文件添加

multiline.pattern: "^# User@Host:"
multiline.negate: true
multiline.match: after

3. 修改filebeat配置文件

vim /etc/filebeat/filebeat.yml
# 在慢日志监控下,添加多行合并配置!添加20-22行,注意yaml格式,开头空2格!
multiline.pattern: "^# User@Host:"
multiline.negate: true
multiline.match: after
 13 # ============================== Filebeat inputs ===============================
 14
 15 filebeat.inputs:
 16 - type: log
 17   enabled: True
 18   paths:
 19     - /usr/local/mysql/data/web-slow.log
 20   multiline.pattern: "^# User@Host:"
 21   multiline.negate: true
 22   multiline.match: after
 23 # Each - is an input. Most options can be set at the input level, so
 24 # you can use different inputs for various configurations.
 25 # Below are the input specific configurations.
 26
 27 # filestream is an input for collecting log messages from files.
 28 #- type: filestream
 29
 30   # Change to true to enable this input configuration.
 31   #  enabled: true
 32
 33   # Paths that should be crawled and fetched. Glob based paths.
 34   #paths:
 35   # - /var/log/chup.log
 36     #- c:\programdata\elasticsearch\logs\*
 37
 38   # Exclude lines. A list of regular expressions to match. It drops the lines that are
 39   # matching any regular expression from the list.
 40   #exclude_lines: ['^DBG']
 41
 42   # Include lines. A list of regular expressions to match. It exports the lines that are
 43   # matching any regular expression from the list.
 44   #include_lines: ['^ERR', '^WARN','sshd']
 45
 46   # Exclude files. A list of regular expressions to match. Filebeat drops the files that
 47   # are matching any regular expression from the list. By default, no files are dropped.
 48   #prospector.scanner.exclude_files: ['.gz$']
 49
 50   # Optional additional fields. These fields can be freely picked
 51   # to add additional information to the crawled log files for filtering
 52   #fields:
 53   #  level: debug
 54   #  review: 1

4. 重启filebeat

systemctl restart filebeat

5. 再次生成慢日志

mysql -uroot -p123456

select * from db1.t1 where id=8;

6. 查看logstash接收到的内容

# 发现:日志从5条变成2条了!
#第一条,无用,想办法将其删除!
 "message" => "# Time: 2021-12-10T07:17:17.100835Z"

#第二条,正是我们想要的内容!
 "message" => "# User@Host: root[root] @ localhost []  Id:     6\n# Query_time: 5.759042  Lock_time: 0.000231 Rows_sent: 1  Rows_examined: 19999999\nSET timestamp=1639120637;\nselect * from db1.t1 where id=8;"
[WARN ] 2022-01-01 22:30:57.556 [defaultEventExecutorGroup-4-2] plain - Relying on default value of `pipeline.ecs_compatibility`, which may change in a future major release of Logstash. To avoid unexpected changes when upgrading Logstash, please explicitly declare your desired ECS Compatibility mode.
#1.第一条
{
           "ecs" => {
        "version" => "1.12.0"
    },
           "log" => {
        "offset" => 877,
          "file" => {
            "path" => "/usr/local/mysql/data/web-slow.log"
        }
    },
          "host" => {
         "architecture" => "x86_64",
                   "id" => "b3856b2bce5c47ab962ede7e592b054c",
                   "ip" => [
            [ 0] "10.0.16.15",
            [ 1] "fe80::5054:ff:fe3c:125f",
            [ 2] "172.17.0.1",
            [ 3] "fe80::42:e8ff:fe51:6b3d",
            [ 4] "10.0.8.5",
            [ 5] "10.244.75.128",
            [ 6] "192.168.250.1",
            [ 7] "fe80::42:f0ff:fe96:2200",
            [ 8] "172.18.0.1",
            [ 9] "fe80::42:aeff:fe33:d359",
            [10] "fe80::40d0:ff:fe10:12eb",
            [11] "fe80::b82e:bdff:fe27:16c5",
            [12] "fe80::c9b:5ff:fe8b:48de",
            [13] "fe80::ecee:eeff:feee:eeee",
            [14] "fe80::ecee:eeff:feee:eeee",
            [15] "fe80::f47c:4aff:fee2:37c",
            [16] "fe80::ecee:eeff:feee:eeee",
            [17] "fe80::f409:3aff:fe34:31e5",
            [18] "fe80::ecee:eeff:feee:eeee",
            [19] "fe80::ecee:eeff:feee:eeee",
            [20] "fe80::a8c6:5ff:fe0a:f9c8",
            [21] "fe80::cc6b:23ff:fe9b:4ac2",
            [22] "fe80::706b:1aff:fe93:c09c",
            [23] "fe80::f8e0:abff:fec5:9520",
            [24] "fe80::da:89ff:fe5b:2491",
            [25] "fe80::642c:c8ff:fe3b:bedc",
            [26] "fe80::481d:16ff:fe20:266"
        ],
                   "os" => {
            "platform" => "ubuntu",
            "codename" => "focal",
              "kernel" => "5.4.0-90-generic",
                "name" => "Ubuntu",
                "type" => "linux",
              "family" => "debian",
             "version" => "20.04 LTS (Focal Fossa)"
        },
                 "name" => "4c16g",
             "hostname" => "4c16g",
        "containerized" => false,
                  "mac" => [
            [ 0] "52:54:00:3c:12:5f",
            [ 1] "02:42:e8:51:6b:3d",
            [ 2] "02:42:f0:96:22:00",
            [ 3] "02:42:ae:33:d3:59",
            [ 4] "42:d0:00:10:12:eb",
            [ 5] "ba:2e:bd:27:16:c5",
            [ 6] "0e:9b:05:8b:48:de",
            [ 7] "ee:ee:ee:ee:ee:ee",
            [ 8] "ee:ee:ee:ee:ee:ee",
            [ 9] "f6:7c:4a:e2:03:7c",
            [10] "ee:ee:ee:ee:ee:ee",
            [11] "f6:09:3a:34:31:e5",
            [12] "ee:ee:ee:ee:ee:ee",
            [13] "ee:ee:ee:ee:ee:ee",
            [14] "aa:c6:05:0a:f9:c8",
            [15] "ce:6b:23:9b:4a:c2",
            [16] "72:6b:1a:93:c0:9c",
            [17] "fa:e0:ab:c5:95:20",
            [18] "02:da:89:5b:24:91",
            [19] "66:2c:c8:3b:be:dc",
            [20] "4a:1d:16:20:02:66"
        ]
    },
         "input" => {
        "type" => "log"
    },
    "@timestamp" => 2022-01-01T14:30:56.539Z,
       "message" => "# Time: 2022-01-01T14:30:52.398839Z",
          "tags" => [
        [0] "beats_input_codec_plain_applied"
    ],
      "@version" => "1",
         "agent" => {
             "version" => "7.16.2",
                  "id" => "edca8f85-bc52-4365-b94b-bdf17eb8209f",
                "name" => "4c16g",
            "hostname" => "4c16g",
                "type" => "filebeat",
        "ephemeral_id" => "3568ddcd-f4a5-499f-84f4-1feac879b4b9"
    },
     "container" => {
        "id" => "web-slow.log"
    }
}
#2.第二条
{
           "ecs" => {
        "version" => "1.12.0"
    },
           "log" => {
         "flags" => [
            [0] "multiline"
        ],
        "offset" => 913,
          "file" => {
            "path" => "/usr/local/mysql/data/web-slow.log"
        }
    },
          "host" => {
         "architecture" => "x86_64",
                   "id" => "b3856b2bce5c47ab962ede7e592b054c",
                   "os" => {
            "platform" => "ubuntu",
            "codename" => "focal",
              "kernel" => "5.4.0-90-generic",
                "name" => "Ubuntu",
                "type" => "linux",
              "family" => "debian",
             "version" => "20.04 LTS (Focal Fossa)"
        },
                   "ip" => [
            [ 0] "10.0.16.15",
            [ 1] "fe80::5054:ff:fe3c:125f",
            [ 2] "172.17.0.1",
            [ 3] "fe80::42:e8ff:fe51:6b3d",
            [ 4] "10.0.8.5",
            [ 5] "10.244.75.128",
            [ 6] "192.168.250.1",
            [ 7] "fe80::42:f0ff:fe96:2200",
            [ 8] "172.18.0.1",
            [ 9] "fe80::42:aeff:fe33:d359",
            [10] "fe80::40d0:ff:fe10:12eb",
            [11] "fe80::b82e:bdff:fe27:16c5",
            [12] "fe80::c9b:5ff:fe8b:48de",
            [13] "fe80::ecee:eeff:feee:eeee",
            [14] "fe80::ecee:eeff:feee:eeee",
            [15] "fe80::f47c:4aff:fee2:37c",
            [16] "fe80::ecee:eeff:feee:eeee",
            [17] "fe80::f409:3aff:fe34:31e5",
            [18] "fe80::ecee:eeff:feee:eeee",
            [19] "fe80::ecee:eeff:feee:eeee",
            [20] "fe80::a8c6:5ff:fe0a:f9c8",
            [21] "fe80::cc6b:23ff:fe9b:4ac2",
            [22] "fe80::706b:1aff:fe93:c09c",
            [23] "fe80::f8e0:abff:fec5:9520",
            [24] "fe80::da:89ff:fe5b:2491",
            [25] "fe80::642c:c8ff:fe3b:bedc",
            [26] "fe80::481d:16ff:fe20:266"
        ],
                 "name" => "4c16g",
             "hostname" => "4c16g",
        "containerized" => false,
                  "mac" => [
            [ 0] "52:54:00:3c:12:5f",
            [ 1] "02:42:e8:51:6b:3d",
            [ 2] "02:42:f0:96:22:00",
            [ 3] "02:42:ae:33:d3:59",
            [ 4] "42:d0:00:10:12:eb",
            [ 5] "ba:2e:bd:27:16:c5",
            [ 6] "0e:9b:05:8b:48:de",
            [ 7] "ee:ee:ee:ee:ee:ee",
            [ 8] "ee:ee:ee:ee:ee:ee",
            [ 9] "f6:7c:4a:e2:03:7c",
            [10] "ee:ee:ee:ee:ee:ee",
            [11] "f6:09:3a:34:31:e5",
            [12] "ee:ee:ee:ee:ee:ee",
            [13] "ee:ee:ee:ee:ee:ee",
            [14] "aa:c6:05:0a:f9:c8",
            [15] "ce:6b:23:9b:4a:c2",
            [16] "72:6b:1a:93:c0:9c",
            [17] "fa:e0:ab:c5:95:20",
            [18] "02:da:89:5b:24:91",
            [19] "66:2c:c8:3b:be:dc",
            [20] "4a:1d:16:20:02:66"
        ]
    },
    "@timestamp" => 2022-01-01T14:30:56.539Z,
       "message" => "# User@Host: root[root] @ localhost []  Id:     5\n# Query_time: 8.120293  Lock_time: 0.000173 Rows_sent: 1  Rows_examined: 19999999\nSET timestamp=1641047452;\nselect * from db1.t1 where id=8;",
         "input" => {
        "type" => "log"
    },
          "tags" => [
        [0] "beats_input_codec_plain_applied"
    ],
      "@version" => "1",
         "agent" => {
        "ephemeral_id" => "3568ddcd-f4a5-499f-84f4-1feac879b4b9",
                  "id" => "edca8f85-bc52-4365-b94b-bdf17eb8209f",
            "hostname" => "4c16g",
                "name" => "4c16g",
                "type" => "filebeat",
             "version" => "7.16.2"
    },
     "container" => {
        "id" => "web-slow.log"
    }
}

最后更新: 2022-02-19 13:05:46