526互联

k8s 1.24.14 Ingress-nginx 的部署

发布时间 2023-05-21 12:39:38作者: kerwin-

前言：

  1. 本次部署使用了高可用的形式，会在每个node节点做亲和性（master不部署），让每一个pod都部署上去，然后加入NGINX去过负载，这样我们之间用NGINX的80端口访问域名就可以了。

  1. MountVolume.SetUp failed for volume "webhook-cert" : secret "ingress-nginx-admission" not found。问题在这个版本好像解决了。

主机 地址 端口
k8s-node01 192.168.80.48 nginx启动端口：3080，负载均衡端口：根据ingress svc自己生成的NodePort的端口
k8s-node02 192.168.80.49 nginx启动端口：3080，负载均衡端口：根据ingress svc自己生成的NodePort的端口
vip 192.168.80.66 访问端口：80

通过 keepalived+nginx 实现 nginx-ingress-controller高可用。

1.安装部署ingress-nginx

1.1.替换镜像

# 查看当前版api版本
kubectl explain Ingress
KIND:     Ingress
VERSION:  networking.k8s.io/v1
....

注：查看ingress和自己本地的k8s版本是否对应上，在GitHub上有表格参考。

mkdir -p /root/ingress && cd /root/ingress
wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.4.0/deploy/static/provider/baremetal/deploy.yaml

cat deploy.yaml | grep image:
image: registry.k8s.io/ingress-nginx/controller:v1.4.0@sha256:34ee929b111ffc7aa426ffd409af44da48e5a0eea1eb2207994d9e0c0882d143
image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343@sha256:39c5b2e3310dc4264d638ad28d9d1d96c4cbb2b2dcfb52368fe4e3c63f61e10f
image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343@sha256:39c5b2e3310dc4264d638ad28d9d1d96c4cbb2b2dcfb52368fe4e3c63f61e10f

# 替换镜像
sed  -i 's#registry.k8s.io/ingress-nginx/controller:v1.4.0@sha256:34ee929b111ffc7aa426ffd409af44da48e5a0eea1eb2207994d9e0c0882d143#registry.cn-hangzhou.aliyuncs.com/imges/controller:v1.4.0#' deploy.yaml
sed  -i 's#registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20220916-gd32f8c343@sha256:39c5b2e3310dc4264d638ad28d9d1d96c4cbb2b2dcfb52368fe4e3c63f61e10f#registry.cn-hangzhou.aliyuncs.com/image-storage/kube-webhook-certgen:v20220916-gd32f8c343#' deploy.yaml

# 后端svc访问改成NodePort
apiVersion: v1
kind: Service
metadata:
  ....
  name: ingress-nginx-controller
  namespace: ingress-nginx
spec:
  ....
  type: NodePort

1.2.ingress高可用配置

1.2.1修改文件和主机打标签

vim deploy.yaml
kind: Deployment	//改为DaemonSet控制器
# replicas: 1		//删除replicas
spec:
  template:
    spec:
      hostNetwork: true	//使用HostNetwork
      nodeSelector:		//修改节点选择，亲和度
        custom/ingress-controller-ready: true

apiVersion: v1
kind: Service
metadata:
  name: ingress-nginx-controller
  namespace: ingress-nginx
  ....
spec:
  ....
  type: NodePort #后端svc访问改成NodePort

# node主机打标签
kubectl label nodes k8s-node01 custom/ingress-controller-ready=true
kubectl label nodes k8s-node02 custom/ingress-controller-ready=true
kubectl taint nodes k8s-master01 node-role.kubernetes.io/master=true:NoSchedule
kubectl taint nodes k8s-master02 node-role.kubernetes.io/master=true:NoSchedule
kubectl taint nodes k8s-master03 node-role.kubernetes.io/master=true:NoSchedule

1.2.2部署ingress

# 部署ingress
kubectl apply -f deploy.yaml

# 查看ingress pod
kubectl get pod -n ingress-nginx 
NAME                                   READY   STATUS      RESTARTS   AGE
ingress-nginx-admission-create-42k7b   0/1     Completed   0          31s
ingress-nginx-admission-patch-j6g9m    0/1     Completed   0          31s
ingress-nginx-controller-8pdz4         1/1     Running     0          31s
ingress-nginx-controller-fdsxb         1/1     Running     0          31s

# 查看ingress svc
kubectl get svc -n ingress-nginx 
NAME                                 TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)                      AGE
ingress-nginx-controller             NodePort   10.98.81.194    <none>        80:30298/TCP,443:32728/TCP   2m54s
ingress-nginx-controller-admission   NodePort   10.97.213.196   <none>        443:31937/TCP                2m54s

2.部署NGINX和keepalived

  • node01、node02操作

apt install nginx keepalived -y
sudo useradd nginx -G www-data

2.1.修改配置

# 修改默认端口为3080
cd /etc/nginx/sites-enabled
cat default
listen 3080 default_server;
listen [::]:3080 default_server;

# 重启nginx
systemctl restart  nginx.service
netstat -lntup  | grep 3080
tcp        0      0 0.0.0.0:3080            0.0.0.0:*               LISTEN      263469/nginx: maste
tcp6       0      0 :::3080                 :::*                    LISTEN      263469/nginx: maste

# 查看ingress本地端口
kubectl get svc  -n ingress-nginx
NAME                                 TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)                      AGE
ingress-nginx-controller             NodePort   10.98.81.194    <none>        80:30298/TCP,443:32728/TCP   13m
ingress-nginx-controller-admission   NodePort   10.97.213.196   <none>        443:31937/TCP                13m

2.2.添加负载

cd /etc/nginx ; cp nginx.conf nginx.conf_bak
cat > nginx.conf <<EOF
load_module /usr/lib/nginx/modules/ngx_stream_module.so;
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;

include /usr/share/nginx/modules/*.conf;

events {
    worker_connections 1024;
}

stream {

    log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';

    access_log  /var/log/nginx/k8s-access.log  main;

    upstream ingress {
       server 192.168.80.45:30298;
       server 192.168.80.46:30298;
       server 192.168.80.47:30298;
       server 192.168.80.48:30298;   # #这里配置成要访问的地址
       server 192.168.80.49:30298;
    }

    server {
       listen 80; #需要监听的端口
       proxy_pass ingress;
    }
}

http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 2048;

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;

    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;

}
EOF

# 检查格式
nginx -t

2.3.keepalived配置

  • k8s-node01

cat > /etc/keepalived/keepalived.conf << EOF
global_defs { 
   notification_email { 
     acassen@firewall.loc 
     failover@firewall.loc 
     sysadmin@firewall.loc 
   } 
   notification_email_from Alexandre.Cassen@firewall.loc  
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id NGINX_MASTER
} 

# 检查脚本
vrrp_script check_nginx {
    script "/etc/keepalived/check_nginx.sh"
}

vrrp_instance VI_1 { 
    state MASTER 
    interface ens33 # 修改为实际网卡名
    virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 
    priority 100    # 优先级，备服务器设置 90 
    advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒 
    authentication { 
        auth_type PASS      
        auth_pass 1111 
    }  
    # 虚拟IP
    virtual_ipaddress { 
        192.168.80.66/24
    } 
    track_script {
        check_nginx
    } 
}
EOF

  • k8s-node02

cat > /etc/keepalived/keepalived.conf << EOF
global_defs { 
   notification_email { 
     acassen@firewall.loc 
     failover@firewall.loc 
     sysadmin@firewall.loc 
   } 
   notification_email_from Alexandre.Cassen@firewall.loc  
   smtp_server 127.0.0.1 
   smtp_connect_timeout 30 
   router_id NGINX_BACKUP
} 

# 检查脚本
vrrp_script check_nginx {
    script "/etc/keepalived/check_nginx.sh"
}

vrrp_instance VI_1 { 
    state BACKUP 
    interface ens33 # 修改为实际网卡名
    virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 
    priority 90     # 优先级，备服务器设置 90 
    advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒 
    authentication { 
        auth_type PASS      
        auth_pass 1111 
    }  
    # 虚拟IP
    virtual_ipaddress { 
        192.168.80.66/24
    } 
    track_script {
        check_nginx
    } 
}
EOF

keepalived 检查脚本（注意脚本内的端口是需要监听的端口）：

cat > /etc/keepalived/check_nginx.sh  << EOF
#!/bin/bash
count=$(ss -antp |grep 80 |egrep -cv "grep|$$")

if [ "$count" -eq 0 ];then
    exit 1
else
    exit 0
fi
EOF

chmod +x /etc/keepalived/check_nginx.sh

重启服务：

systemctl daemon-reload
systemctl start nginx keepalived
systemctl enable nginx keepalived

3.测试ingress

3.1.pod和svc创建

mkdir -p /root/ingress ; cd /root/ingress
cat > /root/ingress/deploy-demo.yaml <<EOF
#创建service为myapp
apiVersion: v1
kind: Service
metadata:
  name: myapp
  namespace: default
spec:
  selector:
    app: myapp
    release: canary
  ports:
  - name: http
    targetPort: 80
    port: 80
---
#创建后端服务的pod
apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp-backend-pod
  namespace: default
spec:
  replicas: 3
  selector:
    matchLabels:
      app: myapp
      release: canary
  template:
    metadata:
      labels:
        app: myapp
        release: canary
    spec:
      containers:
      - name: myapp
        image: ikubernetes/myapp:v2
        ports:
        - name: http
          containerPort: 80
EOF
kubectl apply -f deploy-demo.yaml

# 查看pod启动
kubectl get pod -l app=myapp
NAME                                READY   STATUS    RESTARTS   AGE
myapp-backend-pod-9f9b5bd95-5d487   1/1     Running   0          23m
myapp-backend-pod-9f9b5bd95-k87tc   1/1     Running   0          23m
myapp-backend-pod-9f9b5bd95-vssh7   1/1     Running   0          23m

3.2.ingress创建

cat > /root/ingress/ingress-myapp.yaml <<EOF
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ingress-myapp
  annotations:
    kubernetes.io/ingress.class: "nginx"
spec:
  rules:
  - host: "myapp.magedu.com"
    http:
      paths:
      - pathType: Prefix
        path: "/"
        backend:
          service:
            name: myapp
            port:
              number: 80
EOF
kubectl apply -f ingress-myapp.yaml
 
kubectl get ingress
NAME            CLASS    HOSTS              ADDRESS                       PORTS   AGE
ingress-myapp   <none>   myapp.magedu.com   192.168.80.48,192.168.80.49   80      2m13s

效果：