收集应用特定格式的日志输出至 Elasticsearch 并利用 Kibana展示

1、应用日志收集项说明

root@web01:~# vim /var/log/mall_app.log

[INFO] 2022-06-28 08:08:12 [www.mall.com] - DAU|9136|加入收藏|2022-06-28 01:05:02
[INFO] 2022-06-28 08:08:14 [www.mall.com] - DAU|5035|搜索|2022-06-28 01:07:01
[INFO] 2022-06-28 08:08:15 [www.mall.com] - DAU|669|使用优惠券|2022-06-28 08:05:13
[INFO] 2022-06-28 08:08:19 [www.mall.com] - DAU|2564|搜索|2022-06-28 08:07:08

将日志收集利用 Logstash 进行格式转换后发给 Elasticsearch，并利用Kibana展示

2、配置 Filebeat

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/mall_app.log

output.logstash:
  hosts: ["192.168.1.108:6666"]


root@web01:~# systemctl restart filebeat.service 

3、配置 Logstash

root@logstash01:~# vim /etc/logstash/conf.d/app_filebeat_filter_es.conf

root@logstash01:~# logstash -f /etc/logstash/conf.d/app_filebeat_filter_es.conf -r

{
           "log" => {
          "file" => {
            "path" => "/var/log/mall_app.log"
        },
        "offset" => 334
    },
       "user_id" => "2565",
      "@version" => "1",
         "agent" => {
                  "id" => "2fe34205-12d8-4f37-a3f5-39e0d42f73e6",
             "version" => "7.17.8",
                "type" => "filebeat",
                "name" => "web01.test.com",
        "ephemeral_id" => "8fcd3356-f774-4540-adc8-a4883aa7fe92",
            "hostname" => "web01.test.com"
    },
          "host" => {
                   "id" => "9eaba3e85c9a4e16b66cda021bc16221",
                 "name" => "web01.test.com",
                   "ip" => [
            [0] "192.168.1.105",
            [1] "fe80::20c:29ff:fe6c:ce9f"
        ],
                  "mac" => [
            [0] "00:0c:29:6c:ce:9f"
        ],
             "hostname" => "web01.test.com",
         "architecture" => "x86_64",
                   "os" => {
            "codename" => "focal",
             "version" => "20.04.5 LTS (Focal Fossa)",
                "type" => "linux",
                "name" => "Ubuntu",
              "kernel" => "5.4.0-131-generic",
            "platform" => "ubuntu",
              "family" => "debian"
        },
        "containerized" => false
    },
    "@timestamp" => 2022-06-28T00:07:08.000Z,
        "action" => "搜索",
          "time" => "2022-06-28 08:07:08",
          "tags" => [
        [0] "beats_input_codec_plain_applied"
    ],
         "input" => {
        "type" => "log"
    },
           "ecs" => {
        "version" => "1.12.0"
    }
}

root@web01:~# echo "[INFO] 2022-07-28 08:08:19 [www.mall.com] - DAU|2565|搜索|2022-06-28 08:07:08" >> /var/log/mall_app.log

4、插件查看索引

5、Kibana 创建索引模式

6、Kibana展示