1、环境准备

1.1、安装docker和docker-compose环境

https://www.cnblogs.com/hg-super-man/p/10908220.html

2、安装node_exporter

https://prometheus.io/download/

2.1 二进制安装node_exporter

# 下载node_exporter二进制压缩包
wget https://github.com/prometheus/node_exporter/releases/download/v1.6.1/node_exporter-1.6.1.linux-amd64.tar.gz

# 解压
tar -zxvf node_exporter-1.6.1.linux-amd64.tar.gz

# 移动到/opt
mv node_exporter-1.6.1.linux-amd64 /opt/prometheus/node_exporter

# 更改prometheus用户的文件夹权限
chown -R prometheus:prometheus /opt/prometheus/node_exporter

# 创建systemd 服务
cat > /etc/systemd/system/node_exporter.service << "EOF"
[Unit]
Description=node_exporter
Documentation=https://prometheus.io
After=network.target

[Service]
Type=simple
User=prometheus
Group=prometheus
Restart=on-failure
ExecStart=/opt/prometheus/node_exporter/node_exporter

[Install]
WantedBy=multi-user.target
EOF

# 启动grafana
systemctl daemon-reload
systemctl start node_exporter

# 检查
systemctl status node_exporter

# 加入到开机自启动
systemctl enable node_exporter

2.1.1、 访问地址

http://ip:9100/metrics

2.1.2、修改prometheus配置

在scrape_config这行下面添加如下配置
vi /opt/prometheus/prometheus/prometheus.yml

  # node_exporter配置
  - job_name: "node_exporter"
    scrape_interval: 15s
    static_configs:
    - targets: ["localhost:9100"]
      labels:
        instance: Prometheus服务器

2.1.3、重载prometheus

curl -X POST http://localhost:9090/-/reload

2.2 docker安装node_exporter

mkdir ~/docker/prometheus/docker-compose.yml
vi ~/docker/prometheus/docker-compose.yml

version: '3.3'
services:
  node_exporter:
    image: prom/node-exporter:v1.6.1
    container_name: node_exporter
    restart: always
    volumes:
      - /etc/localtime:/etc/localtime:ro
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.sysfs=/host/sys'
      - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc|rootfs/var/lib/docker)($$|/)'
    ports:
      - '9100:9100'

2.2.1、 检查

cat ~/docker/prometheus/docker-compose.yml

2.2.2 启动

cd ~/docker/prometheus
docker-compose up -d

2.2.3、 访问地址

http://ip:9100/metrics

2.2.4、修改prometheus配置

在scrape_config这行下面添加如下配置
vi /opt/prometheus/prometheus/prometheus.yml

  # node_exporter配置
  - job_name: "node_exporter"
    scrape_interval: 15s
    static_configs:
    - targets: ["localhost:9100"]
      labels:
        instance: Prometheus服务器
    - targets: ["ip:9100"]
      labels:
        instance: test服务器

2.2.5、重载prometheus

curl -X POST http://localhost:9090/-/reload

3、常用监控指标

cpu
内存
磁盘
文件系统
网络

4、触发器设置

4.1、配置

cat >> prometheus/alert.yml <<"EOF"
- name: node-exporter
  rules:
  - alert: HostOutOfMemory
    expr: node_memory_MemAvailable_bytes / node_memory_MeTotal_bytest * 100 < 10
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "主机内存不足，实例：{{ $labels.instance }}"
      description: "内存可用率<10%，当前值：{{ $value }}"

  - alert: HostMemoryUnderMemoryPressure
    expr: rate(node_vmstat_pgmajfault[1m]) > 1000
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "内存压力不足，实例：{{ $labels.instance }}"
      description: "节点内存压力大。重大页面错误率高，当前值：{{ $value }}"

  - alert: HostUnusualNetworkThroughputIn
    expr: sum by (instance) (rate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "异常流入网络吞吐量，实例：{{ $labels.instance }}"
      description: "网络流入流量 > 100 MB/s，当前值：{{ $value }}"

  - alert: HostUnusualNetworkThroughputOut
    expr: sum by (instance) (rate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "异常流出网络吞吐量，实例：{{ $labels.instance }}"
      description: "网络流出流量 > 100 MB/s，当前值：{{ $value }}"

  - alert: HostUnusualDiskReadRate
    expr: sum by (instance) (rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "异常磁盘读取，实例：{{ $labels.instance }}"
      description: "磁盘读取 > 50 MB/s，当前值：{{ $value }}"

  - alert: HostUnusualDiskWriteRate
    expr: sum by (instance) (rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "异常磁盘写入，实例：{{ $labels.instance }}"
      description: "磁盘写入 > 50 MB/s，当前值：{{ $value }}"

  - alert: HostOutOfDiskSpace
    expr: （node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and  ON (instance, device, mountpoint) node_filesystem_readonly == 0
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "磁盘空间不足告警，实例：{{ $labels.instance }}"
      description: "剩余磁盘空间 < 10%，当前值：{{ $value }}"

  - alert: HostDiskWillFillIn24Hours
    expr: （node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and  ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "磁盘空间将在24小时内耗尽，实例：{{ $labels.instance }}"
      description: "以当前写入速率预计磁盘空间在 24 小时内耗尽，当前值：{{ $value }}"

  - alert: HostOutOfInodes
    expr: node_filesystem_files_free{mountpoint = "/"} / node_filesystem_files{mountpoint = "/"} * 100 < 10 and ON (instance, device, mountpoint) node_filesystem_readonly{mountpoint = "/"} == 0
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "磁盘Inodes不足，实例：{{ $labels.instance }}"
      description: "剩余磁盘 inodes < 10%，当前值：{{ $value }}"

  - alert: HostUnusualDiskReadLatency
    expr: rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "异常磁盘读取延迟，实例：{{ $labels.instance }}"
      description: "磁盘读取延迟 > 100ms，当前值：{{ $value }}"

  - alert: HostUnusualDiskWriteLatency
    expr: rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 0.1 and rate(node_disk_writes_completed_total[1m]) > 0
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "异常磁盘写入延迟，实例：{{ $labels.instance }}"
      description: "磁盘写入延迟 > 100ms，当前值：{{ $value }}"

  - alert: high_load
    expr: node_load1 > 4
    for: 2m
    labels:
      severity: page
    annotations:
      summary: "CPU1分钟负载过高，实例：{{ $labels.instance }}"
      description: "CPU1分钟负载>4，已经持续2分钟，当前值：{{ $value }}"

  - alert: HostCpuIsUnderUtilized
    expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode= "idle"}[2m])) * 100) > 80
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: "CPU负载过高，实例：{{ $labels.instance }}"
      description: "CPU负载 > 80%，当前值：{{ $value }}"

  - alert: HostCpuStealNoisyNeighbor
    expr: avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10
    for: 0m
    labels:
      severity: warning
    annotations:
      summary: "CPU窃取率异常，实例：{{ $labels.instance }}"
      description: "CPU窃取率 > 10%，嘈杂的邻居正在扼杀 VM 性能，或者Spot实例可能失去信用，当前值：{{ $value }}"

  - alert: HostSwapIsFillIngUp
    expr: (1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "磁盘swap空间使用率异常，实例：{{ $labels.instance }}"
      description: "磁盘swap空间使用率 > 80%，当前值：{{ $value }}"

  - alert: HostNetworkReceiveErrors
    expr: rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "异常网络接收错误，实例：{{ $labels.instance }}"
      description: "网卡 {{ $labels.device }} 在过去2分钟接收 {{ $value }} 个错误"

  - alert: HostNetworkTransmitErrors
    expr: rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "异常网络传输错误，实例：{{ $labels.instance }}"
      description: "网卡 {{ $labels.device }} 在过去2分钟传输 {{ $value }} 个错误"

  - alert: HostNetworkTransmitErrors
    expr: rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "异常网络传输错误，实例：{{ $labels.instance }}"
      description: "网卡 {{ $labels.device }} 在过去2分钟传输 {{ $value }} 个错误"

  - alert: HostNetworkInterfaceSaturated
    expr: (rate(node_network_receive_bytes_total{device!~"^tap.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*"} > 0.8 < 10000
    for: 1m
    labels:
      severity: warning
    annotations:
      summary: "异常网络接口饱和，实例：{{ $labels.instance }}"
      description: "网卡 {{ $labels.device }} 正在超载，当前值 {{ $value }}"

  - alert: HostConntrackLimit
    expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "异常连接数，实例：{{ $labels.instance }}"
      description: "连接数过大，当前值 {{ $value }}"

  - alert: HostClockSkew
    expr: (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >=0 ) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <=0)
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "异常时钟偏差，实例：{{ $labels.instance }}"
      description: "检测到时钟偏差，时钟不同步，当前值 {{ $value }}"

  - alert: HostClockNotSynchronising
    expr: min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16
    for: 2m
    labels:
      severity: warning
    annotations:
      summary: "时钟不同步，实例：{{ $labels.instance }}"
      description: "时钟不同步"

4.2、检查

cat prometheus/alert.yml

4.3、检查配置

docker exec -it prometheus promtool check config /etc/prometheus/prometheus.yml

4.4、 重写加载配置

curl -X POST http://localhost:9090/-/reload

5、grafana展示node_exporter的数据

因为我们在安装prometheus是，已经在grafana上添加了prometheus的数据源，并导过来id为1860的模板，所以就不需要导入了，直接查看

本栏目推荐文章
• .Net Core 系列： 集成 Consul 实现 服务注册与健康检查
• spark 3.x idea linux远程开发环境搭建
• 字节微服务HTTP框架Hertz使用与源码分析｜拥抱开
• linux环境htop安装
• 【Shell基础】Bash基础与Linux三剑客
• Python教程(23)——Python类中常用的特殊成员
• OpenHarmony 上跑CV 应用 - Windows上搭建 Linux 桌面系统
• Kubernetes Headless服务
• linux 安装最新版本git
• linux环境Python安装

指标 常用 服务器 linux指标 常用 服务器linux 公式 指标 函数 常用 指标 常用kgc 性能 软件测试 指标 常用 指标 服务器 prometheus指标 常用 服务台 指标mtbf mttf 区块 指标 常用 指标 常用 磁盘 性能 指标linux