526互联

k8s 1.26.5 Ingress-nginx 的高可用部署

发布时间 2023-06-11 14:18:46作者: kerwin-

1.安装部署ingress-nginx

本次部署使用了高可用的形式，会在每个node节点做亲和性（master不部署），让每一个pod都部署上去，然后加入NGINX去过负载，这样我们之后用NGINX的80端口访问域名就可以了。

主机 地址 端口
k8s-node01 192.168.80.48 nginx启动端口：3080，负载均衡端口：根据ingress svc自己生成的NodePort的端口
k8s-node02 192.168.80.49 nginx启动端口：3080，负载均衡端口：根据ingress svc自己生成的NodePort的端口
vip 192.168.80.66 访问端口：80

通过 keepalived+nginx 实现 nginx-ingress-controller高可用。

1.1.替换镜像

查看当前版api版本

kubectl explain Ingress
KIND:     Ingress
VERSION:  networking.k8s.io/v1
....

注：查看ingress和自己本地的k8s版本是否对应上，在GitHub上有表格参考。

mkdir -p /root/ingress && cd /root/ingress
curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.6.4/deploy/static/provider/baremetal/deploy.yaml -O deploy.yaml

cat deploy.yaml | grep image:
image: registry.k8s.io/ingress-nginx/controller:v1.6.4@sha256:15be4666c53052484dd2992efacf2f50ea77a78ae8aa21ccd91af6baaa7ea22f
image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343@sha256:39c5b2e3310dc4264d638ad28d9d1d96c4cbb2b2dcfb52368fe4e3c63f61e10f
image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343@sha256:39c5b2e3310dc4264d638ad28d9d1d96c4cbb2b2dcfb52368fe4e3c63f61e10f


# 替换镜像
sed  -i 's#registry.k8s.io/ingress-nginx/controller:v1.6.4@sha256:15be4666c53052484dd2992efacf2f50ea77a78ae8aa21ccd91af6baaa7ea22f#registry.cn-hangzhou.aliyuncs.com/image-storage/controller:v1.6.4#' deploy.yaml
sed  -i 's#registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343@sha256:39c5b2e3310dc4264d638ad28d9d1d96c4cbb2b2dcfb52368fe4e3c63f61e10f#registry.cn-hangzhou.aliyuncs.com/image-storage/kube-webhook-certgen:v20220916-gd32f8c343#' deploy.yaml

注：从外网下载的镜像放在了自己的阿里镜像服务内。

1.2.ingress高可用配置

1.2.1修改文件和主机打标签

vim deploy.yaml
kind: Deployment	#改为DaemonSet控制器
spec:
  template:
    spec:
      nodeSelector:		#修改节点选择，亲和度
        custom/ingress-controller-ready: true

apiVersion: v1
kind: Service
metadata:
  name: ingress-nginx-controller
  namespace: ingress-nginx
  ....
spec:
  ....
  type: NodePort #后端svc访问改成NodePort

# node主机打标签
kubectl label nodes k8s-node01 custom/ingress-controller-ready=true
kubectl label nodes k8s-node02 custom/ingress-controller-ready=true
kubectl taint nodes k8s-master01 node-role.kubernetes.io/master=true:NoSchedule
kubectl taint nodes k8s-master02 node-role.kubernetes.io/master=true:NoSchedule
kubectl taint nodes k8s-master03 node-role.kubernetes.io/master=true:NoSchedule

1.2.2部署ingress

# 部署ingress
kubectl apply -f deploy.yaml

# 查看ingress pod
kubectl get pod -n ingress-nginx 
NAME                                   READY   STATUS      RESTARTS   AGE
ingress-nginx-admission-create-849p4   0/1     Completed   0          61s
ingress-nginx-admission-patch-7qk5c    0/1     Completed   0          61s
ingress-nginx-controller-vblkz         1/1     Running     0          61s
ingress-nginx-controller-zmn49         1/1     Running     0          61s

2.部署NGINX和keepalived

  • node01、node02操作

apt install nginx keepalived -y
sudo useradd nginx -G www-data

2.1.修改配置

# 修改默认端口为3080
cd /etc/nginx/sites-enabled
cat default
listen 3080 default_server;
listen [::]:3080 default_server;

# 重启nginx
systemctl restart  nginx.service
netstat -lntup  | grep 3080
tcp        0      0 0.0.0.0:3080            0.0.0.0:*               LISTEN      263469/nginx: maste
tcp6       0      0 :::3080                 :::*                    LISTEN      263469/nginx: maste

# 查看ingress本地端口
kubectl get svc  -n ingress-nginx
NAME                                 TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                      AGE
ingress-nginx-controller             NodePort    10.101.53.92     <none>        80:30371/TCP,443:32335/TCP   11m
ingress-nginx-controller-admission   ClusterIP   10.101.103.120   <none>        443/TCP                      11m

2.2.添加负载

cd /etc/nginx ; cp nginx.conf nginx.conf_bak
cat > nginx.conf <<"EOF"
load_module /usr/lib/nginx/modules/ngx_stream_module.so;
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;

include /usr/share/nginx/modules/*.conf;

events {
    worker_connections 1024;
}

stream {

    log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';

    access_log  /var/log/nginx/k8s-access.log  main;

    upstream ingress {
       server 192.168.80.48:30371;   # #这里配置成要访问的地址
       server 192.168.80.49:30371;
    }

    server {
       listen 80; #需要监听的端口
       proxy_pass ingress;
    }
}

http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 2048;

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;

    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;

}
EOF

# 检查格式
nginx -t

2.3.keepalived配置

  • k8s-node01

cat > /etc/keepalived/keepalived.conf << EOF
global_defs { 
   notification_email { 
     acassen@firewall.loc 
     failover@firewall.loc 
     sysadmin@firewall.loc 
   } 
   notification_email_from Alexandre.Cassen@firewall.loc  
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id NGINX_MASTER
} 

# 检查脚本
vrrp_script check_nginx {
    script "/etc/keepalived/check_nginx.sh"
}

vrrp_instance VI_NGINX { 
    state MASTER 
    interface ens33 # 修改为实际网卡名
    virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 
    priority 100    # 优先级，备服务器设置 90 
    advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒 
    authentication { 
        auth_type PASS      
        auth_pass 1111 
    }  
    # 虚拟IP
    virtual_ipaddress { 
        192.168.80.66/24
    } 
    track_script {
        check_nginx
    } 
}
EOF

  • k8s-node02

cat > /etc/keepalived/keepalived.conf << EOF
global_defs { 
   notification_email { 
     acassen@firewall.loc 
     failover@firewall.loc 
     sysadmin@firewall.loc 
   } 
   notification_email_from Alexandre.Cassen@firewall.loc  
   smtp_server 127.0.0.1 
   smtp_connect_timeout 30 
   router_id NGINX_BACKUP
} 

# 检查脚本
vrrp_script check_nginx {
    script "/etc/keepalived/check_nginx.sh"
}

vrrp_instance VI_NGINX { 
    state BACKUP 
    interface ens33 # 修改为实际网卡名
    virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 
    priority 90     # 优先级，备服务器设置 90 
    advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒 
    authentication { 
        auth_type PASS      
        auth_pass 1111 
    }  
    # 虚拟IP
    virtual_ipaddress { 
        192.168.80.66/24
    } 
    track_script {
        check_nginx
    } 
}
EOF

keepalived 检查脚本（注意脚本内的端口是需要监听的端口）：

cat > /etc/keepalived/check_nginx.sh  <<"EOF"
#!/bin/bash
count=$(ss -antp |grep 80 |egrep -cv "grep|$$")

if [ "$count" -eq 0 ];then
    exit 1
else
    exit 0
fi
EOF

useradd keepalived_script
passwd keepalived_script
chown -R keepalived_script:keepalived_script /etc/keepalived/check_nginx.sh
chmod +x /etc/keepalived/check_nginx.sh

重启服务：

systemctl daemon-reload
systemctl start nginx keepalived
systemctl enable nginx keepalived
systemctl restart keepalived.service nginx.service

3.测试ingress

3.1.pod和svc创建

mkdir -p /root/ingress ; cd /root/ingress
cat > /root/ingress/deploy-demo.yaml <<EOF
#创建service为myapp
apiVersion: v1
kind: Service
metadata:
  name: myapp
  namespace: default
spec:
  selector:
    app: myapp
    release: canary
  ports:
  - name: http
    targetPort: 80
    port: 80
---
#创建后端服务的pod
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp-backend-pod
  namespace: default
spec:
  replicas: 3
  selector:
    matchLabels:
      app: myapp
      release: canary
  template:
    metadata:
      labels:
        app: myapp
        release: canary
    spec:
      containers:
      - name: myapp
        image: ikubernetes/myapp:v2
        ports:
        - name: http
          containerPort: 80
EOF

启动容器：

kubectl apply -f deploy-demo.yaml

kubectl get pod -l app=myapp
NAME                                READY   STATUS    RESTARTS   AGE
myapp-backend-pod-9f9b5bd95-5d487   1/1     Running   0          23m
myapp-backend-pod-9f9b5bd95-k87tc   1/1     Running   0          23m
myapp-backend-pod-9f9b5bd95-vssh7   1/1     Running   0          23m

3.2.ingress创建

cat > /root/ingress/ingress-myapp.yaml <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-myapp
  annotations:
    kubernetes.io/ingress.class: "nginx"
spec:
  rules:
  - host: "myapp.magedu.com"
    http:
      paths:
      - pathType: Prefix
        path: "/"
        backend:
          service:
            name: myapp
            port:
              number: 80
EOF

创建ingress：

kubectl apply -f ingress-myapp.yaml
 
kubectl get ingress
NAME            CLASS    HOSTS              ADDRESS                       PORTS   AGE
ingress-myapp   <none>   myapp.magedu.com   192.168.80.48,192.168.80.49   80      5m15s

注：Windows本地hosts去绑定下vip地址和ingress定义的域名。 

效果：