搭建ELK日志分析平臺
單機安裝
[root@es-0001 ~]# vim /etc/hosts
192.168.1.41 es-0001
[root@es-0001 ~]# yum install -y java-1.8.0-openjdk elasticsearch
[root@es-0001 ~]# vim /etc/elasticsearch/elasticsearch.yml
55:? network.host: 0.0.0.0
[root@es-0001 ~]# systemctl enable --now elasticsearch
[root@es-0001 ~]# curl http://192.168.1.41:9200/
{
? "name" : "War Eagle",
? "cluster_name" : "elasticsearch",
? "version" : {
? ? "number" : "2.3.4",
? ? "build_hash" : "e455fd0c13dceca8dbbdbb1665d068ae55dabe3f",
? ? "build_timestamp" : "2016-06-30T11:24:31Z",
? ? "build_snapshot" : false,
? ? "lucene_version" : "5.5.0"
? },
? "tagline" : "You Know, for Search"
}
集群安裝
[root@es-0001 ~]# vim /etc/hosts
192.168.1.41 es-0001
192.168.1.42 es-0002
192.168.1.43 es-0003
192.168.1.44 es-0004
192.168.1.45 es-0005
[root@es-0001 ~]# yum install -y java-1.8.0-openjdk elasticsearch
[root@es-0001 ~]# vim /etc/elasticsearch/elasticsearch.yml
17:? cluster.name: my-es
23:? node.name: es-0001 # 本機主機名
55:? network.host: 0.0.0.0
68:? discovery.zen.ping.unicast.hosts: ["es-0001", "es-0002"]
[root@es-0001 ~]# systemctl enable --now elasticsearch
[root@es-0001 ~]# curl http://192.168.1.41:9200/_cluster/health?pretty
{
? "cluster_name" : "my-es",
? "status" : "green",
? "timed_out" : false,
? "number_of_nodes" : 5,
? "number_of_data_nodes" : 5,
? ?... ...
}
插件安裝（head）
安裝 apache，并部署 head 插件
[root@web ~]# yum install -y httpd
[root@web ~]# tar zxf head.tar.gz
[root@web ~]# mv elasticsearch-head /var/www/html/head
[root@web ~]# systemctl enable --now httpd
Created symlink from /etc/systemd/system/multi-user.target.wants/httpd.service to /usr/lib/systemd/system/httpd.service.
[root@es-0001 ~]# vim /etc/elasticsearch/elasticsearch.yml
# 配置文件最后追加
http.cors.enabled : true
http.cors.allow-origin : "*"
http.cors.allow-methods : OPTIONS, HEAD, GET, POST, PUT, DELETE
http.cors.allow-headers : X-Requested-With,X-Auth-Token,Content-Type,Content-Length
[root@es-0001 ~]# systemctl restart elasticsearch.service?
創(chuàng)建索引
指定索引的名稱，指定分片數(shù)量，指定副本數(shù)量
創(chuàng)建索引使用 PUT 方法，創(chuàng)建完成以后通過 head 插件驗證
[root@es-0001 ~]# curl -XPUT -H "Content-Type: application/json" http://es-0001:9200/tedu -d \
'{
? ? "settings":{
? ? ? ?"index":{
? ? ? ? ? "number_of_shards": 5,?
? ? ? ? ? "number_of_replicas": 1
? ? ? ?}
? ? }
}'
增加數(shù)據(jù)
[root@es-0001 ~]# curl -XPUT -H "Content-Type: application/json" http://es-0001:9200/tedu/teacher/1 -d \
'{
? "職業(yè)": "詩人",
? "名字": "李白",
? "稱號": "詩仙",
? "年代": "唐"
}'?
查詢數(shù)據(jù)
[root@es-0001 ~]# curl -XGET http://es-0001:9200/tedu/teacher/1?pretty
修改數(shù)據(jù)
[root@es-0001 ~]# curl -XPOST -H "Content-Type: application/json" \ ? ? ? ? ? ? ? ??
?http://es-0001:9200/tedu/teacher/1/_update -d '{ "doc": {"年代":"公元701"}}'
刪除數(shù)據(jù)
# 刪除一條
[root@es-0001 ~]# curl -XDELETE http://es-0001:9200/tedu/teacher/1
# 刪除索引
[root@es-0001 ~]# curl -XDELETE http://es-0001:9200/tedu
[root@kibana ~]# vim /etc/hosts
192.168.1.41 es-0001
192.168.1.42 es-0002
192.168.1.43 es-0003
192.168.1.44 es-0004
192.168.1.45 es-0005
192.168.1.46 kibana
[root@kibana ~]# yum install -y kibana
[root@kibana ~]# vim /etc/kibana/kibana.yml
02? server.port: 5601
07? server.host: "0.0.0.0"
28? elasticsearch.hosts: ["http://es-0002:9200", "http://es-0003:9200"]
37? kibana.index: ".kibana"
40? kibana.defaultAppId: "home"
113 i18n.locale: "zh-CN"
[root@kibana ~]# systemctl enable --now kibana
[root@logstash ~]# vim /etc/hosts
192.168.1.41 es-0001
192.168.1.42 es-0002
192.168.1.43 es-0003
192.168.1.44 es-0004
192.168.1.45 es-0005
192.168.1.47 logstash
[root@logstash ~]# yum install -y java-1.8.0-openjdk logstash
[root@logstash ~]# ln -s /etc/logstash /usr/share/logstash/config
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {?
? stdin {}
}
filter{ }
output{?
? stdout{}
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
插件與調(diào)試格式
使用json格式字符串測試 ?{"a":"1", "b":"2", "c":"3"}
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {?
? stdin { codec => "json" }
}
filter{ }
output{?
? stdout{ codec => "rubydebug" }
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
官方手冊地址
https://www.elastic.co/guide/en/logstash/current/index.html
input file插件
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {?
? file {
? ? path => ["/tmp/c.log"]
? ? type => "test"
? ? start_position => "beginning"
? ? sincedb_path => "/var/lib/logstash/sincedb"
? }
}
filter{ }
output{?
? stdout{ codec => "rubydebug" }
}
[root@logstash ~]# rm -rf /var/lib/logstash/plugins/inputs/file/.sincedb_*
[root@logstash ~]# /usr/share/logstash/bin/logstash
filter grok插件
正則表達式宏調(diào)用格式: ? ? %{宏名稱:名字}
宏文件路徑
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/logstash-patterns-core-4.1.2/patterns
[root@logstash ~]# echo '192.168.1.252 - - [29/Jul/2020:14:06:57 +0800] "GET /info.html HTTP/1.1" 200 119 "-" "curl/7.29.0"' >/tmp/c.log
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {?
? file {
? ? path => ["/tmp/c.log"]
? ? type => "test"
? ? start_position => "beginning"
? ? sincedb_path => "/dev/null"
? }
}
filter{?
? grok {
? ? match => { "message" => "%{HTTPD_COMBINEDLOG}" }
? }
}
output{?
? stdout{ codec => "rubydebug" }
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
output elasticsearch插件
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {?
? file {
? ? path => ["/tmp/c.log"]
? ? type => "test"
? ? start_position => "beginning"
? ? sincedb_path => "/dev/null"
? }
}
filter{?
? grok {
? ? match => { "message" => "%{HTTPD_COMBINEDLOG}" }
? }
}
output{?
? stdout{ codec => "rubydebug" }
? elasticsearch {
? ? hosts => ["es-0004:9200", "es-0005:9200"]
? ? index => "weblog-%{+YYYY.MM.dd}"
? }
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
[root@logstash ~]# vim /etc/logstash/conf.d/my.conf
input {?
? stdin { codec => "json" }
? file{
? ? path => ["/tmp/c.log"]
? ? type => "test"
? ? start_position => "beginning"
? ? sincedb_path => "/var/lib/logstash/sincedb"
? }
? beats {
? ? port => 5044
? }
}?
filter{?
? grok {
? ? match => { "message" => "%{HTTPD_COMBINEDLOG}" }
? }
}?
output{?
? stdout{ codec => "rubydebug" }
? elasticsearch {
? ? hosts => ["es-0004:9200", "es-0005:9200"]
? ? index => "weblog-%{+YYYY.MM.dd}"
? }
}
[root@logstash ~]# /usr/share/logstash/bin/logstash
web服務(wù)安裝filebeat
[root@web ~]# yum install -y filebeat
[root@web ~]# vim /etc/filebeat/filebeat.yml
24:? enabled: true
28:? - /var/log/httpd/access_log
45:? ? fields:?
46:? ? ? ?my_type: apache
148, 150 注釋掉
161: output.logstash:
163:? ?hosts: ["192.168.1.47:5044"]
180, 181, 182 注釋掉
[root@web ~]# grep -Pv "^\s*(#|$)" /etc/filebeat/filebeat.yml
[root@web ~]# systemctl enable --now filebeat
