安装Prometheus

docker run -d --name=prometheus -p 9090:9090 -v /tmp/prometheus.yml:/etc/prometheus/prometheus.yml prom/prometheus

安装node_exporter

docker run -d --net="host" --pid="host" --userns="host" -v "/:/host:ro,rslave" --name node_exporter -p 9100:9100 --path.rootfs=/host quay.io/prometheus/node-exporter:latest

安装cadvisor

docker run --volume=/:/rootfs:ro --volume=/var/run:/var/run:ro --volume=/sys:/sys:ro --volume=/var/lib/docker/:/var/lib/docker:ro --volume=/dev/disk/:/dev/disk:ro --publish=39200:8080 --detach=true --name=cadvisor --userns=host --privileged --device=/dev/kmsg google/cadvisor

安装grafana(现在2023.12.11时的是安装的grafana-10.2.2)

docker run -d -p 3001:3000 --name=grafana1 -v /apps/data/grafana1:/var/lib/grafana --restart=always grafana/grafana-oss

常用grafana模板id：8919（node_exporter专用），16098（node_exporter专用），193（cadvisor专用）

查询promsql案例
hostname：  node_uname_info{origin_prometheus=~"$origin_prometheus",job=~"$job"} - 0
启动时间：   sum(time() - node_boot_time_seconds{origin_prometheus=~"$origin_prometheus",job=~"$job"})by(instance)/86400
CPU个数：    count(node_cpu_seconds_total{origin_prometheus=~"$origin_prometheus",job=~"$job",mode='system'}) by (instance)
CPU占用率：  (1 - avg(rate(node_cpu_seconds_total{origin_prometheus=~"$origin_prometheus",job=~"$job",mode="idle"}[$interval])) by (instance)) * 100
内存占用率： node_memory_MemTotal_bytes{origin_prometheus=~"$origin_prometheus",job=~"$job"} - 0
负载：      node_load5{origin_prometheus=~"$origin_prometheus",job=~"$job"}

k8s批量部署node_exporter、cadvisor

cat an.sh
#!/bin/bash
yum -y install ansible
sleep 2
echo '[aaa]' >> /etc/ansible/hosts
kubectl get node -o wide|awk 'NR==1 {next} {print $6}' >> /etc/ansible/hosts
kubectl get node -o wide|awk 'NR==1 {next} {print $6}' > an1.txt
#kubectl get node -o wide|awk 'NR==1 {next} {print $1}' > an1.txt
kubectl  get node -o wide |awk 'NR==1 {next} {print $6,$1"-s1"}' > a1.txt
sleep 2
cat   an1.txt |sed  "s/^/'/g"|sed "s/$/\-s1:9100'/g"|tr "\n" ',' > an2.txt
cat   an1.txt |sed  "s/^/'/g"|sed "s/$/\-s1:39200'/g"|tr "\n" ',' > an3.txt
 
#kubectl  get node -o wide |awk 'NR==1 {next} {print $6,$1"-s1"}' > a1.txt




cat >> an.yml <<EOF
---
- name: diskspace_mornitor
  hosts: all
  become: no
  remote_user: root
  gather_facts: no
  tasks:
    - name: check diskspace
      shell: |
        docker run -d --net="host" --pid="host" --userns="host" -v "/:/host:ro,rslave" --name node_exporter -p 9100:9100 quay.io/prometheus/node-exporter:latest --path.rootfs=/host
        sleep 2
        docker run --volume=/:/rootfs:ro --volume=/var/run:/var/run:ro --volume=/sys:/sys:ro --volume=/var/lib/docker/:/var/lib/docker:ro --volume=/dev/disk/:/dev/disk:ro --publish=39200:8080 --detach=true --name=cadvisor --userns=host --privileged --device=/dev/kmsg google/cadvisor
      register: out  
    - debug: var=out.stdout_lines
EOF
sleep 2
ansible-playbook an.yml
sleep 30
ansible all -m shell -a "ss -ntlp|egrep '9100|9200'"  >> an.log
cat an.log