一、方案简述
存储服务组件众多,且容器化多服务实例部署后,日志分散,需要聚合分析,使用 filebeat 来收集节点系统日志、Ceph守护进程实例日志和容器日志,推送至 ELK 集群集中过滤、转换和分析,提高故障排查效率。
二、方案架构图
三、测试环境部署
1、部署单节点ES
容器化部署脚本:
# cat deploy_es.sh #!/bin/bash # docker run \ --name es01-test \ -d --restart=always \ --net elastic \ -p 9200 : 9200 \ -p 9300 : 9300 \ -e "discovery.type=single-node" \ docker.elastic.co/elasticsearch/elasticsearch: 7.17 . 12 |
2、部署Kibana
容器化部署脚本:
# cat deploy_kibana.sh #!/bin/bash # docker run \ --name kib01-test \ -d --restart=always \ --net elastic \ -p 5601 : 5601 \ docker.elastic.co/kibana/kibana: 7.17 . 12 |
3、部署Logstash
安装包:
二进制部署:
rpm -ivh logstash- 7.17 . 12 -x86_64.rpm |
配置示例:
# cat /etc/logstash/conf.d/es-pipeline.conf input { beats { port => 5044 } } output { elasticsearch { index => "ceph-%{[fields][dc]}-%{[fields][env]}-%{+YYYY.MM.dd}" # user => "elastic" # action => "create" # ilm_enabled => true # password => "xxxxx" } } # cat /etc/logstash/conf.d/filter.conf filter { if "syslog" in [tags] { grok { match => { "message" => [ "^%{SYSLOGBASE} %{GREEDYDATA:log_message}" ] } } } else if "ceph-log" in [tags] { grok { match => { "message" => [ "^%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:logsource} %{NOTSPACE:client} %{NOTSPACE:client_ip} %{NOTSPACE} : %{NOTSPACE:program} \[%{NOTSPACE:log_level}\] %{GREEDYDATA:log_message}" ] } } } else if "ceph-audit" in [tags] { grok { match => { "message" => [ "^%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:logsource} %{NOTSPACE:client} %{NOTSPACE:client_ip} %{NOTSPACE} : %{NOTSPACE:program} \[%{NOTSPACE:log_level}\] %{GREEDYDATA:log_message}" ] } } } else if "ceph-mgr" in [tags] { grok { match => { "message" => [ "^%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE}%{SPACE}+%{GREEDYDATA:log_message}" ] } } } else if "ceph-rgw" in [tags] { grok { match => { "message" => [ "^%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE}%{SPACE}+%{GREEDYDATA:log_message}" ] } } } else if "ceph-mds" in [tags] { grok { match => { "message" => [ "^%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE}%{SPACE}+%{GREEDYDATA:log_message}" ] } } } else if "ceph-mon" in [tags] { grok { match => { "message" => [ "^%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE}%{SPACE}+%{GREEDYDATA:log_message}" ] } } } else if "ceph-osd" in [tags] { grok { match => { "message" => [ "^%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE}%{SPACE}+%{GREEDYDATA:log_message}" ] } } } } |
启动服务:
systemctl start logstash systemctl enable logstash |
4、部署Filebeat
安装包:
二进制部署:
rpm -ivh filebeat- 7.17 . 12 -x86_64.rpm |
配置示例:
# cat /etc/filebeat/filebeat.yml filebeat.config: modules: path: ${path.config}/modules.d/*.yml reload.enabled: false setup.template: settings: index.number_of_shards: 1 name: "ceph" pattern: "ceph-*" enabled: true overwrite: true setup.ilm: enabled: false #filebeat.autodiscover: # providers: # - type: docker # hints.enabled: true #processors: #- add_cloud_metadata: ~ #- add_docker_metadata: ~ #- add_host_metadata: ~ #output.elasticsearch: # hosts: 172.16 . 0.1 : 9200 # username: '' # password: '' # index: "ceph-%{[fields.dc]}-%{[fields.env]}-%{+yyyy.MM.dd}" # output.logstash: hosts: [ "172.16.0.1:5044" ] # index: "ceph-%{[fields.dc]}-%{[fields.env]}-%{+yyyy.MM.dd}" setup.kibana: host: 172.16 . 0.1 : 5601 fields: env: prod dc: guangming tags: [ "ceph" , "guangming" , "prod" ] filebeat.inputs: # syslog #- type: filestream # id: syslog-filestream-id # enabled: true # paths: # - /var/log/syslog # - /var/log/messages # fields: # log_source: syslog # tags: [ "syslog" ] # exclude_lines: [ '.*systemd\[\d+\].*' , '.*systemd-resolved\[\d+\].*' , '.*ansible-.*' , '.*filebeat\[\d+\].*' ] # ceph - type: filestream id: ceph-filestream-id enabled: true paths: - /var/log/ceph/ceph.log fields: log_source: ceph-log tags: [ "ceph-log" ] - type: filestream id: ceph-audit-filestream-id enabled: true paths: - /var/log/ceph/ceph.audit.log fields: log_source: ceph-audit tags: [ "ceph-audit" ] - type: filestream id: ceph-mds-filestream-id enabled: true paths: - /var/log/ceph/ceph-mds.*.log fields: log_source: ceph-mds tags: [ "ceph-mds" ] - type: filestream id: ceph-osd-filestream-id enabled: true paths: - /var/log/ceph/ceph-osd.*.log fields: log_source: ceph-osd tags: [ "ceph-osd" ] - type: filestream id: ceph-mon-filestream-id enabled: true paths: - /var/log/ceph/ceph-mon.*.log fields: log_source: ceph-mon tags: [ "ceph-mon" ] - type: filestream id: ceph-mgr-filestream-id enabled: true paths: - /var/log/ceph/ceph-mgr.*.log fields: log_source: ceph-mgr tags: [ "ceph-mgr" ] - type: filestream id: ceph-rgw-filestream-id enabled: true paths: - /var/log/ceph/ceph-client.rgw.*.log fields: log_source: ceph-rgw tags: [ "ceph-rgw" ] - type: filestream id: ceph-volume-filestream-id enabled: true paths: - /var/log/ceph/ceph-volume.log fields: log_source: ceph-volume tags: [ "ceph-volume" ] - type: filestream id: ceph-volume-systemd-filestream-id enabled: true paths: - /var/log/ceph/ceph-volume-systemd.log fields: log_source: ceph-volume-systemd tags: [ "ceph-volume-systemd" ] |
启动服务:
systemctl start filebeat systemctl enable filebeat |
容器化部署:
## 配置示例 # cat filebeat.docker.yml filebeat.config: modules: path: ${path.config}/modules.d/*.yml reload.enabled: false setup.template: settings: index.number_of_shards: 1 name: "ceph" pattern: "ceph-*" enabled: true overwrite: true setup.ilm: enabled: false filebeat.autodiscover: providers: - type: docker hints.enabled: true processors: #- add_cloud_metadata: ~ #- add_docker_metadata: ~ #- add_host_metadata: ~ #output.elasticsearch: # hosts: '172.16.0.1:9200' # username: '' # password: '' # index: "ceph-%{[fields.dc]}-%{[fields.env]}-%{+yyyy.MM.dd}" output.logstash: hosts: [ "172.16.0.1:5044" ] # index: "ceph-%{[fields.dc]}-%{[fields.env]}-%{+yyyy.MM.dd}" setup.kibana: host: "172.16.0.1:5601" filebeat.inputs: #- type: filestream # id: ceph-filestream-id # enabled: true # paths: # - /opt/log/messages # fields: # log_source: syslog # tags: [ "syslog" ] # exclude_lines: [ '.*systemd\[\d+\].*' , '.*systemd-resolved\[\d+\].*' , '.*ansible-.*' , '.*filebeat\[\d+\].*' ] fields: env: pre dc: guangming tags: [ "ceph" , "guangming" , "pre" , "docker" ] ## 部署脚本 # cat deploy_filebeat.sh #!/bin/bash # docker run -d \ --name=filebeat \ --restart=always \ --net=host \ --user=root \ --volume= "$(pwd)/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro" \ --volume= "/var/lib/docker/containers:/var/lib/docker/containers:ro" \ --volume= "/var/run/docker.sock:/var/run/docker.sock:ro" \ --volume= "/var/log:/opt/log:ro" \ docker.elastic.co/beats/filebeat: 7.17 . 12 filebeat -e --strict.perms= false |