526互联

linux-filebeat

发布时间 2023-06-25 21:30:28作者: wh459086748

filebeat

一、部署

1.基于rpm部署

#下载安装包 https://www.elastic.co/
[root@elk101.com ~]# ll
-rw-r--r--  1 root root  34965920 Apr  6 11:19 filebeat-7.17.5-x86_64.rpm

#安装filebeat
[root@elk101.com ~]# rpm -ivh filebeat-7.17.5-x86_64.rpm

#测试
[root@elk101.com ~]# filebeat -h
Usage:
  filebeat [flags]
  filebeat [command]

2.基于二进制部署

#下载安装包
[root@elk103.com ~]# ll
-rw-r--r--  1 root root  35414086 Apr  6 11:19 filebeat-7.17.5-linux-x86_64.tar.gz

#解压软件包
[root@elk103.com ~]# tar xf filebeat-7.17.5-linux-x86_64.tar.gz -C /es/softwares/

#查看软件包
[root@elk103.com ~]# cd /es/softwares/filebeat-7.17.5-linux-x86_64/
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64]# ll
total 114888
-rw-r--r--  1 root root   3780740 Jun 24  2022 fields.yml
-rwxr-xr-x  1 root root 111653152 Jun 24  2022 filebeat
-rw-r--r--  1 root root    170451 Jun 24  2022 filebeat.reference.yml
-rw-------  1 root root      8348 Jun 24  2022 filebeat.yml
drwxr-xr-x  3 root root        15 Jun 24  2022 kibana
-rw-r--r--  1 root root     13675 Jun 24  2022 LICENSE.txt
drwxr-xr-x 76 root root      4096 Jun 24  2022 module
drwxr-xr-x  2 root root      4096 Jun 24  2022 modules.d
-rw-r--r--  1 root root   1987715 Jun 24  2022 NOTICE.txt
-rw-r--r--  1 root root       814 Jun 24  2022 README.md

#创建软链接
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64]# ln -svf  /es/softwares/filebeat-7.17.5-linux-x86_64/filebeat /usr/local/sbin/
‘/usr/local/sbin/filebeat’ -> ‘/es/softwares/filebeat-7.17.5-linux-x86_64/filebeat’

#验证filebeat安装是否成功
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -h
Usage:
  filebeat [flags]
  filebeat [command]

二、filebeat的配置文件

1.基础输入输出

#创建工作目录
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64]# mkdir config

#编写配置文件
[root@elk103.com ~]# cd /es/softwares/filebeat-7.17.5-linux-x86_64/config/
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat 01-stdin-to-console.yaml
# 配置filebeat的输入端
filebeat.inputs:
  # 指定输入端的类型为标准输入
- type: stdin


# 指定filebeat的输出端为console
output.console:
  # 表示输出的内容以漂亮的格式显示
  pretty: true


#启动filebeat的实例
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/01-stdin-to-console.yaml


#输入111信息
#就会输出标准内容

#filebeat输出内容
2023-04-06T19:51:31.398+0800	INFO	[registrar]	registrar/registrar.go:109	States Loaded from registrar: 0
2023-04-06T19:51:31.398+0800	INFO	[crawler]	beater/crawler.go:71	Loading Inputs: 1
2023-04-06T19:51:31.398+0800	INFO	[crawler]	beater/crawler.go:117	starting input, keys present on the config: [filebeat.inputs.0.type]
2023-04-06T19:51:31.398+0800	INFO	[crawler]	beater/crawler.go:148	Starting input (ID: 16876905907669988323)
2023-04-06T19:51:31.398+0800	INFO	[crawler]	beater/crawler.go:106	Loading and starting Inputs completed. Enabled inputs: 1
2023-04-06T19:51:31.398+0800	INFO	[stdin.harvester]	log/harvester.go:309	Harvester started for paths: []	{"harvester_id": "803eb061-bec7-4947-a4ef-31ab3b53edff"}
1111
{
  "@timestamp": "2023-04-06T11:51:36.361Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.17.5"
  },
  "ecs": {
    "version": "1.12.0"
  },
  "host": {
    "name": "elk103.com"
  },
  "agent": {
    "type": "filebeat",
    "version": "7.17.5",
    "hostname": "elk103.com",
    "ephemeral_id": "8f0b3dbc-fa47-475a-80dd-4842d72b120f",
    "id": "1d850d8a-4c3b-4002-9439-16c6760b2bcc",
    "name": "elk103.com"
  },
  "log": {
    "offset": 0,
    "file": {
      "path": ""
    }
  },
  "message": "1111",
  "input": {
    "type": "stdin"
  }
}

注意事项：

filebeat是按行输出的，如果没有换行，是不会输出的

2.filbeat的input插件之tcp案例

[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat 02-tcp-to-console.yaml
filebeat.inputs:
  # 指定类型为tcp
- type: tcp
  # 定义tcp监听的主机和端口
  host: 0.0.0.0:8888

# 指定filebeat的输出端为console
output.console:
  # 表示输出的内容以漂亮的格式显示
  pretty: true


#启动
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/02-tcp-to-console.yaml

#测试
在elk101 nc或者telnet
yum -y install nc telnet

[root@elk101.com ~]# telnet 10.0.0.103 8888
Trying 10.0.0.103...
Connected to 10.0.0.103.
Escape character is '^]'.
1111

#这个时候，在这里任何输入都会输出到filebeat，要退出，要先关闭filebeat

#filebeat输出内容
{
  "@timestamp": "2023-04-06T11:54:17.834Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.17.5"
  },
  "log": {
    "source": {
      "address": "10.0.0.101:49202"
    }
  },
  "input": {
    "type": "tcp"
  },
  "host": {
    "name": "elk103.com"
  },
  "agent": {
    "ephemeral_id": "2bda6e16-2c4e-4e79-bfae-317e2fa9998a",
    "id": "1d850d8a-4c3b-4002-9439-16c6760b2bcc",
    "name": "elk103.com",
    "type": "filebeat",
    "version": "7.17.5",
    "hostname": "elk103.com"
  },
  "ecs": {
    "version": "1.12.0"
  },
  "message": "1111"
}


[root@elk101.com ~]#  echo "AAAAAAAAAAA" | nc 10.0.0.103 8888

#filebeat输出内容
{
  "@timestamp": "2023-04-06T11:55:45.428Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.17.5"
  },
  "input": {
    "type": "tcp"
  },
  "agent": {
    "ephemeral_id": "a77002f1-85de-4668-b576-e5963e8c043b",
    "id": "1d850d8a-4c3b-4002-9439-16c6760b2bcc",
    "name": "elk103.com",
    "type": "filebeat",
    "version": "7.17.5",
    "hostname": "elk103.com"
  },
  "ecs": {
    "version": "1.12.0"
  },
  "host": {
    "name": "elk103.com"
  },
  "message": "AAAAAAAAAAA",
  "log": {
    "source": {
      "address": "10.0.0.101:49204"
    }
  }
}

3.filbeat的input插件之log案例

[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat 03-log-to-console.yaml
filebeat.inputs:
  # 指定输入类型是log
- type: log
  # 指定文件路径
  paths:
    - /tmp/test-filebeat/*.log
    - /tmp/test-filebeat/*/*.json
    # 注意，两个*可以递归匹配
    - /tmp/test-filebeat/**/*.exe

# 指定filebeat的输出端为console
output.console:
  # 表示输出的内容以漂亮的格式显示
  pretty: true


#启动
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/03-log-to-console.yaml

#测试
[root@elk103.com ~]# mkdir -p /tmp/test-filebeat


/tmp/test-filebeat/*.log
#只有/tmp/test-filebeat/*.log会有输出
/tmp/test-filebeat/*/*.json
#只有/test-filebeat下边的第一个子目录下的*.json会有输出
/tmp/test-filebeat/**/*.txt
#只要是/test-filebeat下边的所有txt都会有输出

4.面试题:

下午4点filebeat崩溃，4-5的数据未收集到ES集群。ES集群只有下午5点以后的数据和4点之前的数据。请问如何找回4-5点数据，请说出思路即可。

从es的data目录下找到data/registry/filebeat/log.json，取出时间戳在4-5点的数据。取出读取到的文件偏移量，然后用python或者go语言进行取数即可

5.input的通用字段案例

filebeat input插件的通用字段(common options):
- enabled:
是否启用该组件，有true和false，默认值为true。当设置为false时，表示该input组件不会被加载执行！

- tags:
给每条数据添加一个tags标签列表。

- fields
给数据添加字段。

- fields_under_root
该值默认值为false，将自定义的字段放在一个"fields"的字段中。若设置为true，则将fields的KEY放在顶级字段中。

- processors:
定义处理器，对源数据进行简单的处理。
参考链接:
https://www.elastic.co/guide/en/beats/filebeat/7.17/defining-processors.html

[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat 04-input_common_options-to-console.yaml
filebeat.inputs:
- type: log
  paths:
    - /tmp/test-filebeat/*.log
    - /tmp/test-filebeat/*/*.json
    - /tmp/test-filebeat/**/*.exe
  # 是否启用该类型，默认值为true。
  enabled: false
- type: tcp
  enabled: true
  host: "0.0.0.0:8888"
  # 给数据打标签，会在顶级字段多出来多个标签
  tags: ["Linux","test"]
  # 给数据添加KEY-VALUE类型的字段，默认是放在"fields"中的
  fields:
    school: school1
    class: class01
    classroom: room07
    ip: 219.141.136.10
    port: 13306
  # 若设置为true时，则将fields添加的自定义字段放在顶级字段中，默认值为false。
  fields_under_root: true
  # 定义处理器，过滤指定的数据
  processors:
    # 删除消息是以linux开头的事件(event)
  - drop_event:
      when:
        regexp:
          message: "^linux"
    # 消息包含error内容事件(event)就可以删除自定义字段或者tags。无法删除内置的字段.
  - drop_fields:
      when:
        contains:
          message: "error"
      fields: ["class","tags"]
      ignore_missing: false
    # 修改字段的名称
  - rename:
      fields:
          # 源字段
        - from: "school"
          # 目标字段
          to: "学校"  
        - from: "log"
          to: "日志"
    # 转换数据，将字段的类型转换对应的数据类型，并存放在指定的字段中,本案例将其放在"oldboyedu-linux85"字段中
  - convert:
      fields:
        - {from: "ip", to: "test-filebeat.class07_ip", type: "ip"}
        - {from: "port", to: "test-filebeat.class07_port", type: "integer"}

# 指定filebeat的输出端为console
output.console:
  # 表示输出的内容以漂亮的格式显示
  pretty: true

#启动
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/04-input_common_options-to-console.yaml

#filebeat输出内容
{
  "@timestamp": "2023-04-06T12:22:05.171Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.17.5"
  },
  "input": {
    "type": "tcp"
  },
  "port": 13306,
  "ecs": {
    "version": "1.12.0"
  },
  "agent": {
    "ephemeral_id": "c580f7d4-1220-4527-bbdc-3228c4180895",
    "id": "1d850d8a-4c3b-4002-9439-16c6760b2bcc",
    "name": "elk103.com",
    "type": "filebeat",
    "version": "7.17.5",
    "hostname": "elk103.com"
  },
  "tags": [
    "Linux",
    "test"
  ],
  "学校": "school1",
  "class": "class01",
  "classroom": "room07",
  "ip": "219.141.136.10",
  "日志": {
    "source": {
      "address": "10.0.0.101:49206"
    }
  },
  "test-filebeat": {
    "class07_port": 13306,
    "class07_ip": "219.141.136.10"
  },
  "message": "test",
  "host": {
    "name": "elk103.com"
  }
}

6.包含指定数据采集，排除指定数据采集及json格式数据采集案例

[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat 05-log-to-console.yaml
filebeat.inputs:
- type: log
  paths:
    - /tmp/test-filebeat/*
  # 排除以log结尾的文件
  exclude_files: ['\.log$']
  # 只采集包含指定信息的数据 
  # include_lines: ['linux']
  # 只要包含特定的数据就不采集该事件(event)
  # exclude_lines: ['^linux']
  # 将message字段的json数据格式进行解析，并将解析的结果放在顶级字段中
  json.keys_under_root: true
  # 如果解析json格式失败，则会将错误信息添加为一个"error"字段输出
  json.add_error_key: true

# 指定filebeat的输出端为console
output.console:
  # 表示输出的内容以漂亮的格式显示
  pretty: true

7.使用filebeat采集nginx日志

7.1搭建nginx环境

#添加yum源
cat > /etc/yum.repos.d/nginx.repo <<'EOF'
[nginx-stable]
name=nginx stable repo
baseurl=http://nginx.org/packages/centos/$releasever/$basearch/
gpgcheck=1
enabled=1
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true

[nginx-mainline]
name=nginx mainline repo
baseurl=http://nginx.org/packages/mainline/centos/$releasever/$basearch/
gpgcheck=1
enabled=0
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true
EOF

#安装nginx
[root@elk103.com ~]# yum -y install nginx

7.2 使用filebeat采集nginx的json格式日志

#修改nginx的配置文件
[root@elk103.com ~]# cat /etc/nginx/nginx.conf

user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log notice;
pid        /var/run/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

#注释掉这几行
#    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
#                      '$status $body_bytes_sent "$http_referer" '
#                      '"$http_user_agent" "$http_x_forwarded_for"';
#
#    access_log  /var/log/nginx/access.log  main;
    log_format nginx_json '{"@timestamp":"$time_iso8601",'
                              '"host":"$server_addr",'
                              '"clientip":"$remote_addr",'
                              '"SendBytes":$body_bytes_sent,'
                              '"responsetime":$request_time,'
                              '"upstreamtime":"$upstream_response_time",'
                              '"upstreamhost":"$upstream_addr",'
                              '"http_host":"$host",'
                              '"uri":"$uri",'
                              '"domain":"$host",'
                              '"xff":"$http_x_forwarded_for",'
                              '"referer":"$http_referer",'
                              '"tcp_xff":"$proxy_protocol_addr",'
                              '"http_user_agent":"$http_user_agent",'
                              '"status":"$status"}';

    access_log  /var/log/nginx/access.log  nginx_json;
    sendfile        on;
    #tcp_nopush     on;

    keepalive_timeout  65;

    #gzip  on;

    include /etc/nginx/conf.d/*.conf;
}


#清空access日志内容
[root@elk103.com ~]# > /var/log/nginx/access.log

#启动nginx
[root@elk103.com ~]# systemctl start nginx

#编写配置文件
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat nginx_log_json-to-console.yaml
filebeat.inputs:
- type: log
  paths:
    - /var/log/nginx/access.log*
    #以字段的形式显示
  json.keys_under_root: true
  	#报错的话显示错误信息
  json.add_error_key: true

output.console:
  # 表示输出的内容以漂亮的格式显示
  pretty: true


#启动filebeat
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/nginx_log_json-to-console.yaml 


#访问测试
[root@elk101.com ~]# curl 10.0.0.103

#先查看日志
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat /var/log/nginx/access.log
{"@timestamp":"2023-04-06T20:34:03+08:00","host":"10.0.0.103","clientip":"10.0.0.101","SendBytes":615,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"10.0.0.103","uri":"/index.html","domain":"10.0.0.103","xff":"-","referer":"-","tcp_xff":"-","http_user_agent":"curl/7.29.0","status":"200"}

#查看filebeat输出
{
  "@timestamp": "2023-04-06T12:34:08.789Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.17.5"
  },
  "xff": "-",
  "clientip": "10.0.0.101",
  "http_host": "10.0.0.103",
  "referer": "-",
  "status": "200",
  "upstreamhost": "-",
  "host": {
    "name": "elk103.com"
  },
  "tcp_xff": "-",
  "upstreamtime": "-",
  "SendBytes": 615,
  "input": {
    "type": "log"
  },
  "agent": {
    "name": "elk103.com",
    "type": "filebeat",
    "version": "7.17.5",
    "hostname": "elk103.com",
    "ephemeral_id": "822f8504-f1a4-4247-9955-b47cd3c01ecb",
    "id": "1d850d8a-4c3b-4002-9439-16c6760b2bcc"
  },
  "responsetime": 0,
  "domain": "10.0.0.103",
  "http_user_agent": "curl/7.29.0",
  "uri": "/index.html",
  "log": {
    "offset": 0,
    "file": {
      "path": "/var/log/nginx/access.log"
    }
  },
  "ecs": {
    "version": "1.12.0"
  }
}

8.使用filebeat采集tomcat访问日志和错误日志

8.1 搭建tomcat

#下载tomcat软件包
[root@elk103.com ~]# ll
-rw-r--r--  1 root root  11625808 Apr  6 17:19 apache-tomcat-9.0.73.tar.gz

#解压软件包
[root@elk103.com ~]# tar xf apache-tomcat-9.0.73.tar.gz -C /es/softwares/

#配置环境变量并启动tomcat服务
[root@elk103.com ~]# cat /etc/profile.d/tomcat.sh
#!/bin/bash

export TOMCAT_HOME=/es/softwares/apache-tomcat-9.0.73
export PATH=$PATH:$TOMCAT_HOME/bin
[root@elk103.com ~]# source /etc/profile.d/tomcat.sh

8.2 使用filebeat采集tomcat的json格式日志

#查看filebeat输出#配置tomcat日志格式
[root@elk103.com ~]# cd /es/softwares/apache-tomcat-9.0.73/conf/
[root@elk103.com /es/softwares/apache-tomcat-9.0.73/conf]# cp server.xml{,.bak}
[root@elk103.com /es/softwares/apache-tomcat-9.0.73/conf]# vim server.xml
 #...(切换到行尾修改，大概是在133-149之间)
        <Host name="tomcat.com"  appBase="webapps"
                unpackWARs="true" autoDeploy="true">

                <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
            prefix="tomcat.com_access_log" suffix=".txt"
pattern="{&quot;clientip&quot;:&quot;%h&quot;,&quot;ClientUser&quot;:&quot;%l&quot;,&quot;authenticated&quot;:&quot;%u&quot;,&quot;AccessTime&quot;:&quot;%t&
quot;,&quot;request&quot;:&quot;%r&quot;,&quot;status&quot;:&quot;%s&quot;,&quot;SendBytes&quot;:&quot;%b&quot;,&quot;Query?string&quot;:&quot;%q&quot;,&quot
;partner&quot;:&quot;%{Referer}i&quot;,&quot;http_user_agent&quot;:&quot;%{User-Agent}i&quot;}"/>

          </Host>


#启动tomcat
[root@elk103.com /es/softwares/apache-tomcat-9.0.73/conf]# catalina.sh start
Using CATALINA_BASE:   /es/softwares/apache-tomcat-9.0.73
Using CATALINA_HOME:   /es/softwares/apache-tomcat-9.0.73
Using CATALINA_TMPDIR: /es/softwares/apache-tomcat-9.0.73/temp
Using JRE_HOME:        /es/softwares/jdk1.8.0_291
Using CLASSPATH:       /es/softwares/apache-tomcat-9.0.73/bin/bootstrap.jar:/es/softwares/apache-tomcat-9.0.73/bin/tomcat-juli.jar
Using CATALINA_OPTS:   
Tomcat started.

#使用filebeat采集tomcat日志
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat tomcat_log-to-console.yaml
filebeat.inputs:
- type: log
  paths:
    - /es/softwares/apache-tomcat-9.0.73/logs/tomcat.com_access_log*.txt
  json.keys_under_root: true
  json.add_error_key: true


output.console:
  # 表示输出的内容以漂亮的格式显示
  pretty: true


#访问测试
[root@elk101.com ~]# cat /etc/hosts
10.0.0.103 elk103.com tomcat.com
[root@elk101.com ~]# curl tomcat.com:8080
#查看filebeat输出
{
  "@timestamp": "2023-04-06T12:47:13.872Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.17.5"
  },
  "SendBytes": 615,
  "xff": "-",
  "referer": "-",
  "domain": "tomcat.com",
  "input": {
    "type": "log"
  },
  "clientip": "10.0.0.101",
  "uri": "/index.html",
  "upstreamtime": "-",
  "status": "200",
  "upstreamhost": "-",
  "http_host": "tomcat.com",
  "responsetime": 0,
  "ecs": {
    "version": "1.12.0"
  },
  "agent": {
    "hostname": "elk103.com",
    "ephemeral_id": "822f8504-f1a4-4247-9955-b47cd3c01ecb",
    "id": "1d850d8a-4c3b-4002-9439-16c6760b2bcc",
    "name": "elk103.com",
    "type": "filebeat",
    "version": "7.17.5"
  },
  "log": {
    "offset": 314,
    "file": {
      "path": "/var/log/nginx/access.log"
    }
  },
  "tcp_xff": "-",
  "http_user_agent": "curl/7.29.0",
  "host": {
    "name": "elk103.com"
  }
}

8.3 采集tomcat的错误日志多行匹配案例

[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat tomcat_error_log-to-es.yaml
filebeat.inputs:
- type: log
  paths:
    - /es/softwares/apache-tomcat-9.0.73/logs/catalina*
  multiline.type: pattern
  multiline.pattern: '^\d{2}'
  multiline.negate: true
  multiline.match: after

# 指定输出端为ES集群
output.elasticsearch:
  hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"] 


#测试
[root@elk103.com /es/softwares/apache-tomcat-9.0.73/bin]# ./catalina.sh stop
#先停止服务
#修改配置文件为错误
#启动服务
[root@elk103.com /es/softwares/apache-tomcat-9.0.73/bin]# ./catalina.sh start
[root@elk103.com /es/softwares/apache-tomcat-9.0.73/logs]# tail -20  /es/softwares/apache-tomcat-9.0.73/logs/catalina.out 
		at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
		at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
		at java.lang.reflect.Method.invoke(Method.java:498)
		at org.apache.catalina.startup.Bootstrap.load(Bootstrap.java:307)
		at org.apache.catalina.startup.Bootstrap.main(Bootstrap.java:477)
06-Apr-2023 20:56:49.480 WARNING [main] org.apache.catalina.startup.Catalina.parseServerXml Unable to load server configuration from [/es/softwares/apache-tomcat-9.0.73/conf/server.xml]
	org.xml.sax.SAXParseException; systemId: file:/es/softwares/apache-tomcat-9.0.73/conf/server.xml; lineNumber: 146; columnNumber: 12; The content of elements must consist of well-formed character data or markup.
		at com.sun.org.apache.xerces.internal.parsers.AbstractSAXParser.parse(AbstractSAXParser.java:1243)
		at com.sun.org.apache.xerces.internal.jaxp.SAXParserImpl$JAXPSAXParser.parse(SAXParserImpl.java:644)
		at org.apache.tomcat.util.digester.Digester.parse(Digester.java:1535)
		at org.apache.catalina.startup.Catalina.parseServerXml(Catalina.java:617)
		at org.apache.catalina.startup.Catalina.load(Catalina.java:709)
		at org.apache.catalina.startup.Catalina.load(Catalina.java:746)
		at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
		at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
		at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
		at java.lang.reflect.Method.invoke(Method.java:498)
		at org.apache.catalina.startup.Bootstrap.load(Bootstrap.java:307)
		at org.apache.catalina.startup.Bootstrap.main(Bootstrap.java:477)
06-Apr-2023 20:56:49.480 SEVERE [main] org.apache.catalina.startup.Catalina.start Cannot start server, server instance is not configured


#测试
GET 10.0.0.101:9200/filebeat-7.17.5-2023.04.06-000001/_search
{
    "query":{
        "match_phrase":{
            "message":"at"
        }
    },
    "_source":["message"]
}


#结果
{
                "_index": "filebeat-7.17.5-2023.04.06-000001",
                "_type": "_doc",
                "_id": "VXWsVocBtBzaHDVNQ2_a",
                "_score": 1.0,
                "_source": {
                    "message": "06-Apr-2023 20:56:49.474 SEVERE [main] org.apache.tomcat.util.digester.Digester.fatalError Parse fatal error at line [146] column [12]\n\torg.xml.sax.SAXParseException; systemId: file:/es/softwares/apache-tomcat-9.0.73/conf/server.xml; lineNumber: 146; columnNumber: 12; The content of elements must consist of well-formed character data or markup.\n\t\tat com.sun.org.apache.xerces.internal.util.ErrorHandlerWrapper.createSAXParseException(ErrorHandlerWrapper.java:204)\n\t\tat com.sun.org.apache.xerces.internal.util.ErrorHandlerWrapper.fatalError(ErrorHandlerWrapper.java:178)\n\t\tat com.sun.org.apache.xerces.internal.impl.XMLErrorReporter.reportError(XMLErrorReporter.java:400)\n\t\tat com.sun.org.apache.xerces.internal.impl.XMLErrorReporter.reportError(XMLErrorReporter.java:327)\n\t\tat com.sun.org.apache.xerces.internal.impl.XMLScanner.reportFatalError(XMLScanner.java:1472)\n\t\tat com.sun.org.apache.xerces.internal.impl.XMLDocumentFragmentScannerImpl$FragmentContentDriver.startOfMarkup(XMLDocumentFragmentScannerImpl.java:2637)\n\t\tat com.sun.org.apache.xerces.internal.impl.XMLDocumentFragmentScannerImpl$FragmentContentDriver.next(XMLDocumentFragmentScannerImpl.java:2734)\n\t\tat com.sun.org.apache.xerces.internal.impl.XMLDocumentScannerImpl.next(XMLDocumentScannerImpl.java:605)\n\t\tat com.sun.org.apache.xerces.internal.impl.XMLDocumentFragmentScannerImpl.scanDocument(XMLDocumentFragmentScannerImpl.java:507)\n\t\tat com.sun.org.apache.xerces.internal.parsers.XML11Configuration.parse(XML11Configuration.java:867)\n\t\tat com.sun.org.apache.xerces.internal.parsers.XML11Configuration.parse(XML11Configuration.java:796)\n\t\tat com.sun.org.apache.xerces.internal.parsers.XMLParser.parse(XMLParser.java:142)\n\t\tat com.sun.org.apache.xerces.internal.parsers.AbstractSAXParser.parse(AbstractSAXParser.java:1216)\n\t\tat com.sun.org.apache.xerces.internal.jaxp.SAXParserImpl$JAXPSAXParser.parse(SAXParserImpl.java:644)\n\t\tat org.apache.tomcat.util.digester.Digester.parse(Digester.java:1535)\n\t\tat org.apache.catalina.startup.Catalina.parseServerXml(Catalina.java:617)\n\t\tat org.apache.catalina.startup.Catalina.load(Catalina.java:709)\n\t\tat org.apache.catalina.startup.Catalina.load(Catalina.java:746)\n\t\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\t\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\t\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\t\tat java.lang.reflect.Method.invoke(Method.java:498)\n\t\tat org.apache.catalina.startup.Bootstrap.load(Bootstrap.java:307)\n\t\tat org.apache.catalina.startup.Bootstrap.main(Bootstrap.java:477)"
                }

image-20230406210825075

补充：

[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64]# cat config/tomcat_error_log-to-es.yaml
filebeat.inputs:
- type: log
  paths:
    - /es/softwares/apache-tomcat-9.0.73/logs/catalina*
#  multiline.type: pattern
#  multiline.pattern: '^\d{2}'
#  multiline.negate: true
#  multiline.match: after

# 指定输出端为ES集群
output.elasticsearch:
  hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"] 



#如果不用多行匹配的结果
GET 10.0.0.101:9200/filebeat-7.17.5-2023.04.06-000001/_search
{
    "query":{
        "match_phrase":{
            "message":"at"
        }
    },
    "_source":["message"]
}

image-20230406211017539

练习

1.使用filebeat采集ES服务的启动日志写入ES集群

[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat es_log-to-es.yaml
filebeat.inputs:
- type: log
  paths:
    - /es/logs/es7/linux-es.log

# 指定输出端为ES集群
output.elasticsearch:
  hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"]

2.使用filebeat采集docker日志并写入ES集群;

[root@docker01 /es/softwares/filebeat-7.17.5-linux-x86_64/config]# vim docker_log-to-es.yaml
filebeat.inputs:
- type: docker
  containers.ids:
    - '2a80ae029e3cd467125fb762dd3fcd81bcfe04b7c8f5a3d81cab502b6d8a54ee'
#指定输出端为ES集群
output.elasticsearch:
  hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"]


[root@docker01 ~]# docker inspect 2a80ae029e3c
[
    {
        "Id": "2a80ae029e3cd467125fb762dd3fcd81bcfe04b7c8f5a3d81cab502b6d8a54ee",

9.input多行合并

https://www.elastic.co/guide/en/beats/filebeat/7.17/multiline-examples.html#multiline
#例子1
[beat-logstash-some-name-832-2015.11.28] IndexNotFoundException[no such index]
    at org.elasticsearch.cluster.metadata.IndexNameExpressionResolver$WildcardExpressionResolver.resolve(IndexNameExpressionResolver.java:566)
    at org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.concreteIndices(IndexNameExpressionResolver.java:133)
    at org.elasticsearch.cluster.metadata.IndexNameExpressionResolver.concreteIndices(IndexNameExpressionResolver.java:77)
    at org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction.checkBlock(TransportDeleteIndexAction.java:75)

#log
multiline.type: pattern
multiline.pattern: '^\['
multiline.negate: true
multiline.match: after    
    
 #例子2
 Exception in thread "main" java.lang.NullPointerException
        at com.example.myproject.Book.getTitle(Book.java:16)
        at com.example.myproject.Author.getBookTitles(Author.java:25)
        at com.example.myproject.Bootstrap.main(Bootstrap.java:14)
        
#filestream 
 parsers:
- multiline:
    type: pattern
    pattern: '^[[:space:]]'
    negate: false
    match: after
 
 #log 
multiline.type: pattern
multiline.pattern: '^[[:space:]]'
multiline.negate: false
multiline.match: after


#例子3
Exception in thread "main" java.lang.IllegalStateException: A book has a null property
       at com.example.myproject.Author.getBookIds(Author.java:38)
       at com.example.myproject.Bootstrap.main(Bootstrap.java:14)
Caused by: java.lang.NullPointerException
       at com.example.myproject.Book.getId(Book.java:22)
       at com.example.myproject.Author.getBookIds(Author.java:35)
       ... 1 more
       
 #filestream      
 parsers:
- multiline:
    type: pattern
    pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^Caused by:'
    negate: false
    match: after

#log 
multiline.type: pattern
multiline.pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^Caused by:'
multiline.negate: false
multiline.match: after


#例子4
printf ("%10.10ld  \t %10.10ld \t %s\
  %f", w, x, y, z );
  
#filestream
parsers:
- multiline:
    type: pattern
    pattern: '\\$'
    negate: false
    match: before
    
 #log 
multiline.type: pattern
multiline.pattern: '\\$'
multiline.negate: false
multiline.match: before

#例子5
[2015-08-24 11:49:14,389][INFO ][env                      ] [Letha] using [1] data paths, mounts [[/
(/dev/disk1)]], net usable_space [34.5gb], net total_space [118.9gb], types [hfs]

#filestream
parsers:
- multiline:
    type: pattern
    pattern: '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}'
    negate: true
    match: after
    
#log 
multiline.type: pattern
multiline.pattern: '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}'
multiline.negate: true
multiline.match: after

#例子6
[2015-08-24 11:49:14,389] Start new event
[2015-08-24 11:49:14,395] Content of processing something
[2015-08-24 11:49:14,399] End event
#filestream
parsers:
- multiline:
    type: pattern
    pattern: 'Start new event'
    negate: true
    match: after
    flush_pattern: 'End event'

#log 
multiline.type: pattern
multiline.pattern: 'Start new event'
multiline.negate: true
multiline.match: after
multiline.flush_pattern: 'End event'

10.使用filebeat采集docker日志

#安装docker
[root@elk103.com ~]# ll
-rw-r--r--  1 root root 112195550 Apr  7 10:03 docker-ce-23_0_1.tar.gz
[root@elk103.com ~]# tar xf docker-ce-23_0_1.tar.gz 
[root@elk103.com ~]# cd docker-ce-23/
[root@elk103.com ~/docker-ce-23]# yum -y localinstall *.rpm


#配置docker的镜像加速
[root@elk103.com ~]# cat /etc/docker/daemon.json
{
  "data-root": "/var/lib/docker",
   "registry-mirrors": ["https://tuv7rqqq.mirror.aliyuncs.com","https://hub-mirror.c.1com/","https://docker.mirrors.ustc.edu.cn","https://reg-mirror.qiniu.com"]
}

#启动docker
[root@elk103.com ~]# systemctl enable --now docker


#下载nginx镜像
[root@elk103.com ~]# docker run -dp 88:80 --name mynginx --restart always nginx:1.22.1-alpine

#下载tomcat镜像
[root@elk103.com ~]# docker run -dp 89:8080 --name mytomcat --restart always  tomcat:jre8-alpine


#使用filebeat采集容器日志
[root@elk103.com ~]# cd /es/softwares/filebeat-7.17.5-linux-x86_64/config/
#docker方式采集
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat docker-to-console.yaml
filebeat.inputs:
  # 指定输入类型为docker类型
- type: docker
  # 指定容器的ID
  containers.ids: 
    - '*'


output.console:
  pretty: true


#container采集
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat container-to-console.yaml
filebeat.inputs:
- type: container
  paths: 
    - '/var/lib/docker/containers/*/*.log'

# output.console:
#   pretty: true

output.elasticsearch:
  hosts: ["http://10.0.0.101:9200","http://10.0.0.102:9200","http://10.0.0.103:9200"] 


#采集日志
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/container-to-console.yam

image-20230407202535970

11.filebeat的input类型之filestream实战案例

[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat filestream-to-console.yaml
filebeat.inputs:
  # 指定类型为filestream，在7.16版本中已经弃用log类型
- type: filestream
  enabled: false
  paths:
    - /tmp/test-filebeat/test1.log

- type: filestream
  enabled: false
  paths:
    - /tmp/test-filebeat/test2.json
  # 配置解析
  parsers:
    # 配置json格式解析
    - ndjson:
       # 将错误消息记录到error字段中
       add_error_key: true
       # 如果解析的json格式字段和filebeat内置的顶级字段冲突，则覆盖，默认是不覆盖的。
       overwrite_keys: true
       # 将message解析的字段放入一个自定义的字段下。若不指定该字段，则默认解析的键值对会在顶级字段.
       #target: test-filebeat

- type: filestream
  enabled: true
  paths:
    - /tmp/test-filebeat/demo.log
  parsers:
    - multiline:
        type: count
        count_lines: 4
    - ndjson:
       add_error_key: true
       overwrite_keys: true
       target: test-filebeat-demo

output.console:
  pretty: true

[root@elk103.com /tmp/test-filebeat]# cat test1.log
1111
2222
3333
4444
[root@elk103.com /tmp/test-filebeat]# cat test2.json
{"name":"test01","age":"25"}

[root@elk103.com /tmp/test-filebeat]# cat demo.log
{
"name":"stu01",
"age":18
}
{
"name":"stu02",
"age":20
}


#采集数据
{
  "@timestamp": "2023-04-07T12:32:50.283Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.17.5"
  },
  "input": {
    "type": "filestream"
  },
  "ecs": {
    "version": "1.12.0"
  },
  "host": {
    "name": "elk103.com"
  },
  "agent": {
    "ephemeral_id": "bfd69c53-78b1-4a28-958a-9d7aa967bc6e",
    "id": "a7447022-b8dd-47a3-8496-27558631e7c2",
    "name": "elk103.com",
    "type": "filebeat",
    "version": "7.17.5",
    "hostname": "elk103.com"
  },
  "log": {
    "file": {
      "path": "/tmp/test-filebeat/demo.log"
    },
    "flags": [
      "multiline"
    ],
    "offset": 27
  },
  "test-filebeat-demo": {
    "age": 18,
    "name": "stu01"
  }
}
{
  "@timestamp": "2023-04-07T12:32:50.283Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "7.17.5"
  },
  "log": {
    "offset": 56,
    "file": {
      "path": "/tmp/test-filebeat/demo.log"
    },
    "flags": [
      "multiline"
    ]
  },
  "test-filebeat-demo": {
    "name": "stu02",
    "age": 20
  },
  "input": {
    "type": "filestream"
  },
  "ecs": {
    "version": "1.12.0"
  },
  "host": {
    "name": "elk103.com"
  },
  "agent": {
    "id": "a7447022-b8dd-47a3-8496-27558631e7c2",
    "name": "elk103.com",
    "type": "filebeat",
    "version": "7.17.5",
    "hostname": "elk103.com",
    "ephemeral_id": "bfd69c53-78b1-4a28-958a-9d7aa967bc6e"
  }
}

12.将数据写入到本地文件案例

[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat stdin-to-file.yaml
filebeat.inputs:
- type: stdin

# 指定输出的类型为本地文件
output.file:
  # 指定文件存储的路径
  path: "/tmp/test-filebeat"
  # 指定文件的名称
  filename: stdin.log

#采集数据
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/stdin-to-file.yaml 
输入111111


[root@elk103.com /tmp/test-filebeat]# cat stdin.log 
{"@timestamp":"2023-04-07T12:35:04.439Z","@metadata":{"beat":"filebeat","type":"_doc","version":"7.17.5"},"log":{"offset":0,"file":{"path":""}},"input":{"type":"stdin"},"ecs":{"version":"1.12.0"},"host":{"name":"elk103.com"},"agent":{"ephemeral_id":"9483e7aa-6983-4ab6-b5fa-26676489aa7b","id":"a7447022-b8dd-47a3-8496-27558631e7c2","name":"elk103.com","type":"filebeat","version":"7.17.5","hostname":"elk103.com"},"message":"11111"}

13.写入数据到ES集群

[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat log-to-es.yaml
filebeat.inputs:
- type: filestream
  enabled: true
  paths:
    - /tmp/test-filebeat/shopping.json
  parsers:
    - multiline:
        type: count
        count_lines: 7
    - ndjson:
       add_error_key: true
       overwrite_keys: true

# 将日志输出到ES集群
output.elasticsearch:
  # 指定ES集群地址
  hosts: 
  - "http://10.0.0.101:9200"
  - "http://10.0.0.102:9200"
  - "http://10.0.0.103:9200"
  # 指定索引
  index: "linux-es-shopping-%{+yyyy.MM.dd}"

# 禁用索引声明管理周期，若不禁用则自动忽略自定义索引名称
setup.ilm.enabled: false
# 设置索引模板的名称
setup.template.name: "linux-es-shopping"
# 指定索引模板的匹配模式
setup.template.pattern: "linux-es-shopping-*"
# 是否覆盖原有的索引模板
setup.template.overwrite: true
# 设置索引模板
setup.template.settings:
  # 指定分片数量为8
  index.number_of_shards: 8
  # 指定副本数量为0
  index.number_of_replicas: 0
  
  #采集数据
  [root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/log-to-es.yaml

建议：

一般先output到控制台，测试正确了，在往es上写

image-20230407204249567

14.将多个数据源写入到ES集群不同索引

[root@elk103.com /tmp/test-filebeat]# cat testlines3.log
{
"name":"testlines3"
}


[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/config]# cat filestream-to-es.yaml
filebeat.inputs:
- type: filestream
  enabled: true
  tags: "test2"
  paths:
    - /tmp/test-filebeat/test2.json
  parsers:
    - ndjson:
       add_error_key: true


- type: filestream
  enabled: true
  tags: "testlines3"
  paths:
    - /tmp/test-filebeat/testlines3.log
  parsers:
    - multiline:
        type: count
        count_lines: 3


- type: filestream
  enabled: true
  tags: "demo"
  paths:
    - /tmp/test-filebeat/demo.log
  parsers:
    - multiline:
        type: count
        count_lines: 4
    - ndjson:
       add_error_key: true
       overwrite_keys: true
       target: test-filebeat-demo


output.elasticsearch:
  hosts: 
  - "http://10.0.0.101:9200"
  - "http://10.0.0.102:9200"
  - "http://10.0.0.103:9200"
  indices:
     - index: "linux-es-test2-%{+yyyy.MM.dd}"
       when.contains:
         tags: "test2"
     - index: "linux-es-testlines3-%{+yyyy.MM.dd}"
       when.contains:
         tags: "testlines3"
     - index: "linux-es-demo-%{+yyyy.MM.dd}"
       when.contains:
         tags: "demo"

setup.ilm.enabled: false
setup.template.name: "linux-es"
setup.template.pattern: "linux-es-*"
setup.template.overwrite: true
setup.template.settings:
  index.number_of_shards: 3
  index.number_of_replicas: 0


#采集数据
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64]# filebeat -e -c config/filestream-to-es.yaml

#注意setup.template.name和setup.template.pattern别和现有的冲突

image-20230407205542328

15.filebeat模块使用

#有点不太好使，先弃用吧，修改nginx配置文件格式吧
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/config]# modules-to-logstash.yaml
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false

# output.logstash:
#   hosts: ["10.0.0.101:7777"]
#
#
output.console:
  pretty: true
  
 
 [root@elk103.oldboyedu.com /oldboyedu/softwares/filebeat-7.17.5-linux-x86_64]# ll modules.d |grep nginx
-rw-r--r-- 1 root root   784 Jun 24  2022 nginx.yml.disablednginx.yml.disabled

[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64]# cd modules.d
[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/modules.d]# mv nginx.yml.disabled nginx.yml



#直接mv一下,把nginx.yml.disabled 改为nginx.yml



[root@elk103.com /es/softwares/filebeat-7.17.5-linux-x86_64/modules.d]# cat nginx.yml
# Module: nginx
# Docs: https://www.elastic.co/guide/en/beats/filebeat/7.17/filebeat-module-nginx.html

- module: nginx
  # Access logs
  access:
    enabled: true

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    #修改这一行
    var.paths: ["/tmp/test-filebeat/access.log"]

  # Error logs
  error:
    enabled: false

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    #var.paths:

  # Ingress-nginx controller logs. This is disabled by default. It could be used in Kubernetes environments to parse ingress-nginx logs
  ingress_controller:
    enabled: false

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    #var.paths:


./filebeat modules list   # 显示所有模块
./filebeat modules -h  # 显示modules帮助命令
./filebeat -h  # 显示帮助命令
./filebeat modules enable nginx  # 启用指定模块
./filebeat -e  # 前台执行