环境:
OS:Centos 7
说明:
logstash本身不自带logstash-input-mongodb插件,需要自行安装。
1.安装编译工具
yum install git
yum install gem
2.源码编译
[root@localhost]#git clone https://github.com/phutchins/logstash-input-mongodb.git
[root@localhost]#cd logstash-input-mongodb
[root@localhost logstash-input-mongodb]# gem build *.gemspec
Successfully built RubyGem
Name: logstash-input-mongodb
Version: 0.4.1
File: logstash-input-mongodb-0.4.1.gem
这个时候会生成logstash-input-mongodb-0.4.1.gem文件
[root@localhost logstash-input-mongodb]# ls -al
total 40
drwxr-xr-x. 6 root root 234 Nov 2 04:13 .
drwxr-xr-x. 3 root root 36 Nov 2 04:12 ..
-rw-r--r--. 1 root root 720 Nov 2 04:12 DEVELOPER.md
-rw-r--r--. 1 root root 38 Nov 2 04:12 Gemfile
-rw-r--r--. 1 root root 2335 Nov 2 04:12 Gemfile.lock
drwxr-xr-x. 8 root root 163 Nov 2 04:12 .git
drwxr-xr-x. 3 root root 22 Nov 2 04:12 lib
-rw-r--r--. 1 root root 594 Nov 2 04:12 LICENSE
-rw-r--r--. 1 root root 11776 Nov 2 04:13 logstash-input-mongodb-0.4.1.gem
-rw-r--r--. 1 root root 1255 Nov 2 04:12 logstash-input-mongodb.gemspec
-rw-r--r--. 1 root root 33 Nov 2 04:12 Rakefile
-rw-r--r--. 1 root root 3453 Nov 2 04:12 README.md
drwxr-xr-x. 3 root root 20 Nov 2 04:12 spec
drwxr-xr-x. 2 root root 31 Nov 2 04:12 test
3.查看当前安装的插件
[root@localhost bin]# cd /opt/logstash-6.8.5/bin
[root@localhost bin]#./logstash-plugin list
4.安装
[root@localhost bin]# cd /opt/logstash-6.8.5/bin
[root@localhost bin]# ./logstash-plugin install /soft/mongo2es/logstash-input-mongodb/logstash-input-mongodb-0.4.1.gem
提示报错:
ERROR: Something went wrong when installing /soft/mongo2es/logstash-input-mongodb/logstash-input-mongodb-0.4.1.gem, message: execution expired
ERROR: Something went wrong when installing /soft/mongo2es/logstash-input-mongodb/logstash-input-mongodb-0.4.1.gem, message: Socket closed
解决办法:
重新运行,该命令要执行很久,至少一个小时
5.logstash同步配置文件
[root@localhost config]# more sync_mongo_es.conf
input {
mongodb {
codec => "json"
parse_method => "simple"
uri => 'mongodb://192.168.1.108:29001/db_pushmsg'
placeholder_db_dir => '/opt/logstash-6.8.5/db_dir'
placeholder_db_name =>'app_message_all.db'
collection => 'app_message_all'
}
}
# 该部分被注释,表示filter是可选的
filter {
mutate {
remove_field => ["host","@version","logdate","log_entry","@timestamp","mongo_id"]
}
mutate {
rename => { "_id" => "uid" }
}
}
output {
elasticsearch {
hosts => ["http://192.168.1.109:19200"]
user => "elastic"
password => "elastic123"
index => "index_app_message_all"
##document_type => "%{[@metadata][_type]}"
##document_id => "%{[@metadata][_id]}"
}
}