20.接收kafka日志

0. 环境¶

# 当前内存：0.64G
# 启动logstash后内存：1.12G
# 大概使用内存：0.5G！！！

# 当前logstash服务器：10.0.8.1
# kafka服务器：10.0.8.2
# es服务器：10.0.8.5
# 业务：10.0.8.3

1. 安装logstash¶

# 10.0.8.1
cd /home/admin
ls
dpkg -i logstash*

2. 编写logstash配置文件¶

# 接收kafka的消息，然后发送到elasticsearch
#1.复制配置文件
cd /etc/logstash
ls
cp logstash-sample.conf conf.d/kafka-to-elasticsearch.conf

#2.修改配置文件
cd conf.d
vim kafka-to-elasticsearch.conf
# 写入一下内容：input:从kafka拿数据；output：将数据发送给elasticsearch

input {
  kafka {
    bootstrap_servers => "10.0.8.2:9092" # 消费者们,如果多台kafka机器的话，直接在这里加
    topics => "chupeng"
    codec => "json"
    consumer_threads => 5
    decorate_events => true
  }
}

output {
  elasticsearch {
    hosts => ["10.0.8.5:9200"]
    index => "tomcat-log-%{+YYYY-MM-DD}"
    codec => "json"
  }
}

3. 测试启动logstash¶

/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/kafka-to-elasticsearch.conf -t

4. 正式启动logstash¶

/usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/kafka-to-elasticsearch.conf 

# 启动时，发现之前有启动过的实例，先关闭它
ps aux|grep logstash
kill -9 `进程号`

# 如果报错不认识的zabbix，则设置hosts解析！
cat /etc/hosts
echo "10.0.8.1 ali" >>/etc/hosts
echo "10.0.8.2 zabbix" >>/etc/hosts
echo "10.0.8.3 k8s-master" >>/etc/hosts
echo "10.0.8.5 4c16g" >>/etc/hosts
cat /etc/hosts

5. 后台启动logstash¶

# 没问题后，后台启动！
cd /tmp
nohup /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/kafka-to-elasticsearch.conf &

6. 确认logstash是否启动成功¶

jobs

7. 刷新页面，产生日志¶

# 10.0.8.3:8080
# 访问或者刷新页面，产生访问日志

8. 查看elasticsearch索引¶

# 10.0.8.5
curl 10.0.8.5:9200/_cat/indices

root@4c16g:~# curl 10.0.8.5:9200/_cat/indices
green  open .kibana_task_manager_7.16.2_001       1n0PxyQoSJKKmUs20b79qA 1 0  17 73642  17.4mb  17.4mb
green  open .kibana_7.16.2_001                    LCun_7ggQiekuppoURCj8w 1 0 412    15   2.4mb   2.4mb
green  open .apm-agent-configuration              kQzGyZJxSSaotJkeBbpVbQ 1 0   0     0    226b    226b
yellow open filebeat-7.16.2-2022.01.01            q8xdwWFyRzayYohqua64og 1 1  25     0 194.8kb 194.8kb
green  open .tasks                                rDc3gYaqTtiYoUe7TPlBig 1 0   4     0  21.4kb  21.4kb
green  open .geoip_databases                      vp77wdCxR6ersomYKR7XIg 1 0  44     5  40.8mb  40.8mb
yellow open 10.0.8.4-nginx-accesslog_2021.12.31   mjvv2a2GRB-VS6zAi-R1VA 1 1  66     0 189.9kb 189.9kb
yellow open 10.0.8.1-apache2-accesslog_2022.01.01 NVjpSyKHTa-mmGVJ3fbS3w 1 1 292     0   529kb   529kb
yellow open 10.0.8.1-apache2-accesslog_2022.01.02 H2juWrgeQ6m8VUWwhnr_LA 1 1 168     0 442.9kb 442.9kb
green  open .apm-custom-link                      NwEzmXWMRb-Lnnw_Y8C33Q 1 0   0     0    226b    226b
yellow open tomcat-log-2022-01-02                 NXaDZ0qPSvmo8GEuzrV68Q 1 1   1     0  12.4kb  12.4kb  # 出现！
green  open .async-search                         9RuXcU90T1OMOX-AZ6YDZA 1 0   0     0    256b    256b
yellow open 10.0.8.4-nginx-accesslog_2022.01.01   LmH5-VlxRgCnEROuEH7BUA 1 1 145     0 350.3kb 350.3kb
yellow open 10.0.8.1-apache2-accesslog_2021.12.31 P07lUSMvQCy_erHOT5lW6w 1 1  88     0 228.5kb 228.5kb
yellow open 10.0.8.4-nginx-accesslog_2022.01.02   MnukySvGSnik9HXFzFczPQ 1 1  49     0 168.1kb 168.1kb

9. 打开kibana查看¶

# 10.0.8.5:5601
# 添加新的tomcat索引！

10. 绘制图表¶

# 多访问几次页面，产生多次访问日志，便于绘图！！！
# 10.0.8.3:8080/book

最后更新: 2022-02-19 13:05:46