#在filebeat服务器安装nginx并且配置json格式的日志 root@ubuntu:~# cat /etc/nginx/nginx.conf user www-data; worker_processes auto; pid /run/nginx.pid; include /etc/nginx/modules-enabled/*.conf; events { worker_connections 768; } http { sendfile on; tcp_nopush on; tcp_nodelay on; keepalive_timeout 65; types_hash_max_size 2048; include /etc/nginx/mime.types; default_type application/octet-stream; ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; # Dropping SSLv3, ref: POODLE ssl_prefer_server_ciphers on; log_format access_json '{"@timestamp":"$time_iso8601",'#这里是为了让nginx的日志时间覆盖掉es里面的timestamp的时间 '"host":"$server_addr",' '"clientip":"$remote_addr",' '"size":$body_bytes_sent,' '"responsetime":$request_time,' '"upstreamtime":"$upstream_response_time",' '"upstreamhost":"$upstream_addr",' '"http_host":"$host",' '"uri":"$uri",' '"domain":"$host",' '"xff":"$http_x_forwarded_for",' '"referer":"$http_referer",' '"tcp_xff":"$proxy_protocol_addr",' '"http_user_agent":"$http_user_agent",' '"status":"$status"}'; access_log /var/log/nginx/access_json.log access_json ; error_log /var/log/nginx/error.log; gzip on; include /etc/nginx/conf.d/*.conf; include /etc/nginx/sites-enabled/*; } #重启nginx systemctl restart nginx.service #在10.0.0.47安装redis yum install -y redis #配置redis vim /etc/redis/redis.conf bind 0.0.0.0 requirepass 123456 #启动redis [root@redis ~]#systemctl restart redis #记得暂时关闭rsyslog因为日志太多 systemctl stop rsyslog.service syslog.socket #配置filebeat收集nginx的日志和超级系统的日志 cat >/etc/filebeat/filebeat.yml<<'EOF' filebeat.inputs: - type: log enabled: true paths: - /var/log/nginx/access_json.log json.keys_under_root: true #默认False会将json数据存储至message，改为true则会独立message外存储 json.overwrite_keys: true #设为true,覆盖默认的message字段，使用自定义json格式中的key tags: ["nginx-access"] - type: log enabled: true paths: - /var/log/nginx/error.log tags: ["nginx-error"] - type: log enabled: true paths: - /var/log/syslog tags: ["syslog"] output.redis: hosts: ["10.0.0.47:6379"] password: "123456" db: "0" key: "filebeat" #所有日志都存放在key名称为filebeat的列表中，llen filebeat可查看长度，即日志记录数 EOF #logstash是根据tags来区分不同的日志发送到es实现建立不同的索引，所以filebeat要配置tage #清空nginx的日志 root@ubuntu:~# >/var/log/nginx/access_json.log root@ubuntu:~# >/var/log/nginx/error.log #启动filebeat systemctl restart filebeat.service #查看redis的key root@10:/var/log# redis-cli -a 123456 Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe. 127.0.0.1:6379> KEYS * 1) "filebeat" 127.0.0.1:6379> type filebeat list 127.0.0.1:6379> llen filebeat (integer) 17047 127.0.0.1:6379> llen filebeat (integer) 17047 127.0.0.1:6379> llen filebeat (integer) 17047 #取出日志信息查看 127.0.0.1:6379> lpop filebeat "{\"@timestamp\":\"2023-08-10T14:05:34.000Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.17.5\"},\"size\":612,\"status\":\"200\",\"responsetime\":0,\"domain\":\"10.0.0.74\",\"xff\":\"-\",\"uri\":\"/index.nginx-debian.html\",\"clientip\":\"10.0.0.74\",\"tags\":[\"nginx-access\"],\"input\":{\"type\":\"log\"},\"upstreamtime\":\"-\",\"upstreamhost\":\"-\",\"http_user_agent\":\"curl/7.68.0\",\"referer\":\"-\",\"agent\":{\"version\":\"7.17.5\",\"hostname\":\"ubuntu\",\"ephemeral_id\":\"b956afe8-085f-4b7f-914e-533880abf106\",\"id\":\"12aede24-e6de-40b9-a884-583d5996bd74\",\"name\":\"ubuntu\",\"type\":\"filebeat\"},\"tcp_xff\":\"-\",\"log\":{\"offset\":39904,\"file\":{\"path\":\"/var/log/nginx/access_json.log\"}},\"ecs\":{\"version\":\"1.12.0\"},\"http_host\":\"10.0.0.74\",\"host\":{\"name\":\"ubuntu\"}}" 127.0.0.1:6379> lpop filebeat "{\"@timestamp\":\"2023-08-10T14:05:34.000Z\",\"@metadata\":{\"beat\":\"filebeat\",\"type\":\"_doc\",\"version\":\"7.17.5\"},\"ecs\":{\"version\":\"1.12.0\"},\"uri\":\"/index.nginx-debian.html\",\"size\":612,\"referer\":\"-\",\"status\":\"200\",\"domain\":\"10.0.0.74\",\"responsetime\":0,\"input\":{\"type\":\"log\"},\"host\":{\"name\":\"ubuntu\"},\"xff\":\"-\",\"clientip\":\"10.0.0.74\",\"upstreamhost\":\"-\",\"tcp_xff\":\"-\",\"upstreamtime\":\"-\",\"http_host\":\"10.0.0.74\",\"tags\":[\"nginx-access\"],\"agent\":{\"hostname\":\"ubuntu\",\"ephemeral_id\":\"b956afe8-085f-4b7f-914e-533880abf106\",\"id\":\"12aede24-e6de-40b9-a884-583d5996bd74\",\"name\":\"ubuntu\",\"type\":\"filebeat\",\"version\":\"7.17.5\"},\"http_user_agent\":\"curl/7.68.0\",\"log\":{\"offset\":40222,\"file\":{\"path\":\"/var/log/nginx/access_json.log\"}}}" #触发nginx的访问日志在检查redis的filebeat的key的数量 root@10:~# curl 10.0.0.74 -I HTTP/1.1 200 OK Server: nginx/1.18.0 (Ubuntu) Date: Sat, 12 Aug 2023 18:25:04 GMT Content-Type: text/html Content-Length: 612 Last-Modified: Wed, 09 Aug 2023 15:29:54 GMT Connection: keep-alive ETag: "64d3b0f2-264" Accept-Ranges: bytes root@10:~# curl 10.0.0.74/XX -I HTTP/1.1 404 Not Found Server: nginx/1.18.0 (Ubuntu) Date: Sat, 12 Aug 2023 18:25:06 GMT Content-Type: text/html Content-Length: 162 Connection: keep-alive #检查redis 127.0.0.1:6379> llen filebeat (integer) 17045 127.0.0.1:6379> llen filebeat (integer) 17045 127.0.0.1:6379> llen filebeat (integer) 17047 127.0.0.1:6379> llen filebeat (integer) 17047 127.0.0.1:6379> llen filebeat (integer) 17047 #配置logstash收集日志输出日志到es cat >/etc/logstash/conf.d/redis_geoip_to_es.conf.conf<<'EOF' input { redis { host => '10.0.0.47' port => "6379" password => "123456" db => "0" #在0号数据库抓取日志 data_type => 'list' #数据类型为list key => "filebeat" #把收集到的日志传输给filebeat的key } } filter { if "nginx-access" in [tags] { # geoip { # source => "clientip" # target => "geo" # } } } output { if "syslog" in [tags] { elasticsearch { hosts => ["10.0.0.70:9200","10.0.0.71:9200","10.0.0.72:9200"] index => "syslog-%{+YYYY.MM.dd}" } } if "nginx-access" in [tags] { elasticsearch { hosts => ["10.0.0.70:9200","10.0.0.71:9200","10.0.0.72:9200"] index => "logstash-nginx-accesslog-%{+YYYY.MM.dd}" template_overwrite => true } } if "nginx-error" in [tags] { elasticsearch { hosts => ["10.0.0.70:9200","10.0.0.71:9200","10.0.0.72:9200"] index => "logstash-nginx-errorlog-%{+YYYY.MM.dd}" template_overwrite => true } } } EOF #做语法检查 logstash -f /etc/logstash/conf.d/redis_geoip_to_es.conf.conf -t #启动 logstash -f /etc/logstash/conf.d/redis_geoip_to_es.conf.conf -r #检查redis的key root@10:/var/log# redis-cli -a 123456 Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe. 127.0.0.1:6379> llen filebeat (integer) 0 127.0.0.1:6379> llen filebeat (integer) 0 #已经被logstash提取完毕 #在网页检查es集群
