Linux部署Prometheus+Alertmanager并配置钉钉告警

1. ## Request timeout
2. # timeout: 5s
4. ## Uncomment following line in order to write template from scratch (be careful!)
5. #no_builtin_template: true
7. ## Customizable templates path
8. templates:
9. - '/usr/local/dingtalk/template/dingtalk.tmpl'
11. ## You can also override default template using `default_message`
12. ## The following example to use the 'legacy' template from v0.3.0
13. #default_message:
14. # title: '{{ template "legacy.title" . }}'
15. # text: '{{ template "legacy.content" . }}'
17. ## Targets, previously was known as "profiles"
18. targets:
19. webhook:
20. url: https://oapi.dingtalk.com/robot/send?access_token=输入自己的Token
21. # secret for signature
22. secret: 加签内容

1. {{ define "__subject" }}
2. [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]
3. {{ end }}
6. {{ define "__alert_list" }}{{ range . }}
7. ---
8. {{ if .Labels.owner }}@{{ .Labels.owner }}{{ end }}
10. 告警状态：{{ .Status }}
12. 告警级别：{{ .Labels.severity }}
14. 告警类型：{{ .Labels.alertname }}
16. 告警主机：{{ .Labels.instance }}
18. 告警详情：{{ .Annotations.description }}
20. 告警时间：{{ (.StartsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}
21. {{ end }}{{ end }}
23. {{ define "__resolved_list" }}{{ range . }}
24. ---
25. {{ if .Labels.owner }}@{{ .Labels.owner }}{{ end }}
26. 告警状态：{{ .Status }}
28. 告警级别：{{ .Labels.severity }}
30. 告警类型：{{ .Labels.alertname }}
32. 告警主机：{{ .Labels.instance }}
34. 告警详情：{{ .Annotations.description }}
36. 告警时间：{{ (.StartsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}
38. 恢复时间：{{ (.EndsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}
39. {{ end }}{{ end }}
41. {{ define "default.title" }}
42. {{ template "__subject" . }}
43. {{ end }}
45. {{ define "default.content" }}
46. {{ if gt (len .Alerts.Firing) 0 }}
47. **Prometheus故障告警**
48. {{ template "__alert_list" .Alerts.Firing }}
49. ---
50. {{ end }}
53. {{ if gt (len .Alerts.Resolved) 0 }}
54. **Prometheus故障恢复**
55. {{ template "__resolved_list" .Alerts.Resolved }}
56. {{ end }}
57. {{ end }}
59. {{ define "ding.link.title" }}{{ template "default.title" . }}{{ end }}
60. {{ define "ding.link.content" }}{{ template "default.content" . }}{{ end }}
61. {{ template "default.title" . }}
62. {{ template "default.content" . }}

1. [Unit]
2. Description=prometheus-webhook-dingtalk
3. After=network-online.target
5. [Service]
6. Type=simple
7. ExecStart=/usr/local/dingtalk/prometheus-webhook-dingtalk --config.file=/usr/local/dingtalk/config.yml
8. Restart=on-failure
10. [Install]
11. WantedBy=multi-user.target

1. [Unit]
2. Description=node_exporter
3. Documentation=https://prometheus.io/
4. After=network.target
6. [Service]
7. Type=simple
8. User=root
9. ExecStart=/usr/local/node_exporter/node_exporter
10. Restart=on-failure
12. [Install]
13. WantedBy=multi-user.target

1. global:
2. alerting:
3. alertmanagers:
4. - static_configs:
5. - targets:
6. - '192.168.11.199:9093'
7. rule_files:
8. - 'rules/node.yml'
9. scrape_configs:
10. - job_name: "node-exporter"
11. static_configs:
12. - targets:
13. - "192.168.11.199:9100"

1. groups:
2. - name: Node Resource Monitoring
3. rules:
4. - alert: Node实例已宕机
5. expr: up{instance =~ ".*:9100"} == 0
6. for: 15s
7. labels:
8. user: root
9. severity: Emergency
10. annotations:
11. summary: "{{ $labels.instance }} 客户端已停止运行,请尽快处理!"
12. description: "Node_exporter客户端已停止运行超过15s,当前状态：{{ $value }} "

1. [Unit]
2. Description=prometheus
3. Documentation=https://prometheus.io/docs/introduction/overview
4. Wants=network-online.target
5. After=network.target
7. [Service]
8. Type=simple
9. User=root
10. ExecStart=/usr/local/prometheus/prometheus --config.file=/usr/local/prometheus/prometheus.yml --web.enable-lifecycle
11. ExecReload=/bin/kill -HUP $MAINPID
12. ExecStop=/bin/kill -s SIGTERM $MAINPID
13. Restart=on-failure
15. [Install]
16. WantedBy=multi-user.target

1. [Unit]
2. Description=alertmanager
3. Documentation=https://github.com/prometheus/alertmanager
4. After=network.target
6. [Service]
7. Type=simple
8. User=root
9. ExecStart=/usr/local/alertmanager/alertmanager --config.file=/usr/local/alertmanager/alertmanager.yml
10. Restart=on-failure
12. [Install]
13. WantedBy=multi-user.target

1. global:
2. # 每2分钟检查一次是否恢复
3. resolve_timeout: 2m
4. # route用来设置报警的分发策略
5. route:
6. # 采用哪个标签来作为分组依据
7. group_by: ['alertname']
8. # 组告警等待时间。也就是告警产生后等待10s，如果有同组告警一起发出
9. group_wait: 10s
10. # 两组告警的间隔时间
11. group_interval: 10s
12. # 重复告警的间隔时间，减少相同微信告警的发送频率
13. repeat_interval: 1h
14. # 设置默认接收人
15. receiver: 'webhook'
16. receivers:
17. - name: 'webhook'
18. webhook_configs:
19. - url: 'http://localhost:8060/dingtalk/webhook/send'
20. send_resolved: true

1. groups:
2. - name: CPU
3. rules:
4. - alert: CPU使用率过高
5. expr: (100 - (avg by (instance) (irate(node_cpu{mode="idle"}[5m])) * 100)) > 80
6. for: 1m
7. labels:
8. severity: Warning
9. annotations:
10. summary: "{{ $labels.instance }} CPU使用率过高"
11. description: "{{ $labels.instance }}: CPU使用率超过80%，当前使用率({{ $value }})."

1. groups:
2. - name: Memory
3. rules:
4. - alert: 内存使用率过高
5. expr: (node_memory_MemTotal - (node_memory_MemFree+node_memory_Buffers+node_memory_Cached )) / node_memory_MemTotal * 100 > 80
6. for: 1m # 告警持续时间，超过这个时间才会发送给alertmanager
7. labels:
8. severity: Warning
9. annotations:
10. summary: "{{ $labels.instance }} 内存使用率过高"
11. description: "{{ $labels.instance }}：内存使用率超过80%，当前使用率({{ $value }})."

转自https://www.i7ti.cn/1497.html    感谢作者提供的灵感

本栏目推荐文章
• spark 3.x idea linux远程开发环境搭建
• linux环境htop安装
• 【Shell基础】Bash基础与Linux三剑客
• OpenHarmony 上跑CV 应用 - Windows上搭建 Linux 桌面系统
• linux 安装最新版本git
• linux环境Python安装
• 纪念一下，在国产麒麟linux下跑dotnet8，运行起来了
• linux 磁盘清理
• Linux的守护进程 [补档-2023-08-10]
• Linux的信号管理 [补档-2023-07-30]

Alertmanager Prometheus Linuxalertmanager prometheus linux企业 alertmanager prometheus邮件linux alertmanager prometheus linux prometheus-alertmanager alertmanager prometheus alertmanager prometheus grafana alertmanager prometheus amp prometheus-alertmanager alertmanager prometheus webhook-dingtalk alertmanager prometheus dingtalk prometheus服务器linux