I. ELK installation
1. Software architecture: filebeat----elasticsearch----kibana+sentinel -- (mail and nails)
2. Software download address: https://www.elastic.co/cn/downloads/past-releases #(this tutorial uses 6.2.4)
3.elasticsearch installation
[root@VM_0_7_centos ~]# egrep -v "^$|^#" /opt/app/elasticsearch-6.2.4/config/elasticsearch.yml
cluster.name: globalglb-elk node.name: globalglb network.host: 0.0.0.0 http.port: 9200 http.cors.enabled: true http.cors.allow-origin: "*"
4.kibana installation
[root@VM_0_10_centos ~]# egrep -v "^$|^#" /opt/app/kibana-6.2.4-linux-x86_64/config/kibana.yml
server.port: 5601
server.host: "10.9.0.10"
elasticsearch.url: "http://10.9.0.7:9200"
sentinl:
settings:
email:
active: true
user: elkmonitor@126.com
password: YOUxin2019
host: smtp.126.com
ssl: false
report:
active: true
4.filebeat configuration information
#cat ffilebeat.yml
filebeat.prospectors:
########################
- input_type: log
paths:
- /opt/app/logs/evolut-api-gateway/evolut-api-gateway.log
#json.keys_under_root: true
#json.overwrite_keys: true
fields:
index: 'prd-evolut-api-gateway'
exclude_lines: ['^$']
multiline:
pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
negate: true
match: after
#############################
- input_type: log
paths:
- /opt/app/logs/evolut-file-service/evolut-file-service.log
#json.keys_under_root: true
#json.overwrite_keys: true
fields:
index: 'prd-evolut-file-service'
exclude_lines: ['^$']
multiline:
pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
negate: true
match: after
#####################################
- input_type: log
paths:
- /opt/app/logs/evolut-admin/evolut-admin.log
#json.keys_under_root: true
#json.overwrite_keys: true
fields:
index: 'prd-evolut-admin'
exclude_lines: ['^$']
multiline:
pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
negate: true
match: after
##################################
- input_type: log
paths:
- /opt/app/logs/evolut-insurance/evolut-insurance.log
#json.keys_under_root: true
#json.overwrite_keys: true
fields:
index: 'prd-evolut-insurance'
exclude_lines: ['^$']
multiline:
pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
negate: true
match: after
#########################
- input_type: log
paths:
- /opt/app/logs/evolut-message/evolut-message.log
#json.keys_under_root: true
#json.overwrite_keys: true
fields:
index: 'prd-evolut-message'
exclude_lines: ['^$']
multiline:
pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
negate: true
match: after
####################
- input_type: log
paths:
- /opt/app/logs/evolut-schedule/evolut-schedule.log
#json.keys_under_root: true
#json.overwrite_keys: true
fields:
index: 'prd-evolut-schedule'
exclude_lines: ['^$']
multiline:
pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
negate: true
match: after
############
- input_type: log
paths:
- /opt/app/logs/evolut-user/evolut-user.log
#json.keys_under_root: true
#json.overwrite_keys: true
fields:
index: 'prd-evolut-user'
exclude_lines: ['^$']
multiline:
pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
negate: true
match: after
####################
####################
- input_type: log
paths:
- /opt/app/logs/evolut-esign/evolut-esign.log
#json.keys_under_root: true
#json.overwrite_keys: true
fields:
index: 'prd-evolut-esign'
exclude_lines: ['^$']
multiline:
pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
negate: true
match: after
###################
output.elasticsearch:
hosts: ["10.9.0.7:9200"]
indices:
- index: "prd-evolut-file-service-%{+YYYY.MM.dd}"
when.contains:
fields:
index: "prd-evolut-file-service"
- index: "prd-evolut-api-gateway-%{+YYYY.MM.dd}"
when.contains:
fields:
index: "prd-evolut-api-gateway"
- index: "prd-evolut-admin-%{+YYYY.MM.dd}"
when.contains:
fields:
index: "prd-evolut-admin"
- index: "prd-evolut-insurance-%{+YYYY.MM.dd}"
when.contains:
fields:
index: "prd-evolut-insurance"
- index: "prd-evolut-message-%{+YYYY.MM.dd}"
when.contains:
fields:
index: "prd-evolut-message"
- index: "prd-evolut-schedule-%{+YYYY.MM.dd}"
when.contains:
fields:
index: "prd-evolut-schedule"
- index: "prd-evolut-user-%{+YYYY.MM.dd}"
when.contains:
fields:
index: "prd-evolut-user"
- index: "prd-evolut-esign-%{+YYYY.MM.dd}"
when.contains:
fields:
index: "prd-evolut-esign"Note: filebeat monitors multiple files according to their home indexes
II. Configure kibana+sentnl email and nail alarm
1. Log in to the console and directly import the following code. Modify it according to the modification.
{
"actions": {
"Mail alarm": {
"name": "Log anomaly",
"throttle_period": "0h2m0s",
"email_html": {
"stateless": false,
"subject": "evolut-api-gateway Modular--ERROR Journal",
"priority": "medium",
"html": "<p><i>Hi,Dear colleagues, please note that {{payload.hits.total}} Error messages, please check and handle!!</i>.</p>\n<div style=\"color:grey;\">\n <hr />\n</div>\n<div>\n<br>{{#payload.hits.hits}} <li style='color:red'><b>source:</b> {{_source.source}} </li><br><li><b>message</b>: {{_source.message}}</li><br><br>{{/payload.hits.hits}} \n</div>",
"to": "xiong@xxx.com",
"from": "e@126.com"
}
},
"Nail alarm template": {
"name": "webhook Give an alarm",
"throttle_period": "0h2m0s",
"webhook": {
"priority": "medium",
"stateless": false,
"method": "POST",
"host": "oapi.dingtalk.com",
"port": "443",
"path": "/robot/send?access_token=bdf86156bcded8b10727ceff898b943ef726baaebd797f760336",
"body": "{\r\n \"msgtype\": \"markdown\",\r\n \"at\": {\r\n \"isAtAll\": \"True\"\r\n },\r\n \"markdown\": {\r\n \"title\": \"Exception message\",\r\n \"text\": \" evolut-api-gateway Modular-Error log: \\n {{#payload.hits.hits}} {{_source.message}} \r\n{{/payload.hits.hits}}\"\r\n }\r\n}",
"params": {
"watcher": "{{watcher.title}}",
"payload_count": "{{payload.hits.total}}"
},
"headers": {
"Content-Type": "application/json"
},
"message": "Abnormal production environment",
"use_https": true
}
}
},
"input": {
"search": {
"request": {
"index": [
"prd-evolut-api-gateway*"
],
"body": {
"query": {
"bool": {
"must": {
"match": {
"message": "ERROR"
}
},
"filter": {
"range": {
"@timestamp": {
"gte": "now-5m/m",
"lte": "now/m",
"format": "epoch_millis"
}
}
}
}
},
"size": 2,
"aggs": {
"dateAgg": {
"date_histogram": {
"field": "@timestamp",
"time_zone": "Asia/Shanghai",
"interval": "1m",
"min_doc_count": 1
}
}
}
}
}
}
},
"condition": {
"script": {
"script": "payload.hits.total >= 1"
}
},
"transform": {},
"trigger": {
"schedule": {
"later": "every 2 minutes"
}
},
"disable": false,
"report": false,
"title": "evolut-api-gateway"
}Email alarm content
Alarm mail
Nail warning
Login pin - new group - select robot
