案例(部署mysql)本节使用 StatefulSet 控制器部署一个 MySQL 集群，然后进行宕机测试，观察集群是否可以正常恢复使用并且不丢失数据。 实现的集群有如下特征： 是一个主从复制的 MySQL 集群 1个主节点， 多个从节点 从节点能够水平扩展 所有的写操作，只能在主节点上执行 读操作可以在所有节点上执行 #准备xtrabackup镜像 [root@localhost7C mysql]# docker pull registry.cn-hangzhou.aliyuncs.com/hxpdocker/xtrabackup:1.0 [root@localhost7C mysql]# docker tag c415dbd7af07 harobr.zzhz.com/linux39/xtrabackup:1.0 [root@localhost7C mysql]# docker push harobr.zzhz.com/linux39/xtrabackup:1.0 #准备mysql 镜像 [root@localhost7C mysql]# docker pull registry.cn-hangzhou.aliyuncs.com/acs-sample/mysql:5.7 [root@localhost7C mysql]# docker tag 54fec2fa59d0 harobr.zzhz.com/linux39/mysql:5.7 [root@localhost7C mysql]# docker push harobr.zzhz.com/linux39/mysql:5.7
2.安装nfs服务器，基于PV作为后端存储 [root@localhost7B ]# cat /etc/exports /data/k8sdata *(rw,no_root_squash) [root@localhost7B ~]# mkdir /data/k8sdata/magedu/mysql-datadir-1 [root@localhost7B ~]# mkdir /data/k8sdata/magedu/mysql-datadir-2 [root@localhost7B ~]# mkdir /data/k8sdata/magedu/mysql-datadir-3 [root@localhost7B ~]# mkdir /data/k8sdata/magedu/mysql-datadir-4 [root@localhost7B ]# systemctl restart nfs-server.service
[root@localhost7C mysql]# tree
├── mysql-configmap.yaml
├── mysql-services.yaml
├── mysql-statefulset.yaml
└── pv
└── mysql-persistentvolume.yaml
[root@localhost7C mysql]# cat pv/mysql-persistentvolume.yaml --- apiVersion: v1 kind: PersistentVolume metadata: name: mysql-datadir-1 namespace: magedu spec: capacity: storage: 50Gi accessModes: - ReadWriteOnce nfs: path: /data/k8sdata/magedu/mysql-datadir-1 server: 192.168.80.110 --- apiVersion: v1 kind: PersistentVolume metadata: name: mysql-datadir-2 namespace: magedu spec: capacity: storage: 50Gi accessModes: - ReadWriteOnce nfs: path: /data/k8sdata/magedu/mysql-datadir-2 server: 192.168.80.110 --- apiVersion: v1 kind: PersistentVolume metadata: name: mysql-datadir-3 namespace: magedu spec: capacity: storage: 50Gi accessModes: - ReadWriteOnce nfs: path: /data/k8sdata/magedu/mysql-datadir-3 server: 192.168.80.110 --- apiVersion: v1 kind: PersistentVolume metadata: name: mysql-datadir-4 namespace: magedu spec: capacity: storage: 50Gi accessModes: - ReadWriteOnce nfs: path: /data/k8sdata/magedu/mysql-datadir-4 server: 192.168.80.110
#mysql主从的配置文件 [root@localhost7C mysql]# cat mysql-configmap.yaml apiVersion: v1 kind: ConfigMap metadata: name: mysql labels: app: mysql data: master.cnf: | # Apply this config only on the master. [mysqld] log-bin log_bin_trust_function_creators=1 lower_case_table_names=1 slave.cnf: | # Apply this config only on slaves. [mysqld] super-read-only log_bin_trust_function_creators=1
@两个 Service [root@localhost7C mysql]# cat mysql-services.yaml # StatefulSet成员的稳定DNS项的无头服务。 apiVersion: v1 kind: Service metadata: name: mysql labels: app: mysql spec: ports: - name: mysql port: 3306 clusterIP: None selector: app: mysql --- #用于连接到任何MySQL实例进行读取的客户端服务。 #对于写入，您必须连接到master：mysql-0.mysql。 apiVersion: v1 kind: Service metadata: name: mysql-read labels: app: mysql spec: # type: NodePort ports: - name: mysql port: 3306 # targetPort: 3306 # nodePort: 33306 selector: app: mysql
#yaml清单 [root@localhost7C mysql]# cat mysql-statefulset.yaml apiVersion: apps/v1 kind: StatefulSet metadata: name: mysql spec: selector: matchLabels: app: mysql serviceName: mysql replicas: 2 template: metadata: labels: app: mysql spec: #定义初始化容器 initContainers: - name: init-mysql image: harbor.zzhz.com/linux39/mysql:5.7 command: - bash - "-c" - | set -ex # Generate mysql server-id from pod ordinal index. [[ `hostname` =~ -([0-9]+)$ ]] || exit 1 ordinal=${BASH_REMATCH[1]} echo [mysqld] > /mnt/conf.d/server-id.cnf # Add an offset to avoid reserved server-id=0 value. echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf # Copy appropriate conf.d files from config-map to emptyDir. if [[ $ordinal -eq 0 ]]; then cp /mnt/config-map/master.cnf /mnt/conf.d/ else cp /mnt/config-map/slave.cnf /mnt/conf.d/ fi volumeMounts: - name: conf #调用emptyDir mountPath: /mnt/conf.d - name: config-map #调用configmap mountPath: /mnt/config-map #定义初始化容器 - name: clone-mysql image: harbor.zzhz.com/linux39/xtrabackup:1.0 command: - bash - "-c" - | set -ex # Skip the clone if data already exists. [[ -d /var/lib/mysql/mysql ]] && exit 0 # Skip the clone on master (ordinal index 0). [[ `hostname` =~ -([0-9]+)$ ]] || exit 1 ordinal=${BASH_REMATCH[1]} [[ $ordinal -eq 0 ]] && exit 0 # Clone data from previous peer. ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql # Prepare the backup. xtrabackup --prepare --target-dir=/var/lib/mysql volumeMounts: - name: data #调用PVC模版 mountPath: /var/lib/mysql subPath: mysql - name: conf #调用emptyDir mountPath: /etc/mysql/conf.d #定义容器 containers: - name: mysql image: harbor.zzhz.com/linux39/mysql:5.7 env: - name: MYSQL_ALLOW_EMPTY_PASSWORD value: "1" ports: - name: mysql containerPort: 3306 volumeMounts: - name: data #调用PVC模版 mountPath: /var/lib/mysql subPath: mysql - name: conf #调用emptyDir mountPath: /etc/mysql/conf.d resources: requests: cpu: 500m memory: 1Gi livenessProbe: exec: command: ["mysqladmin", "ping"] initialDelaySeconds: 30 periodSeconds: 10 timeoutSeconds: 5 readinessProbe: exec: # Check we can execute queries over TCP (skip-networking is off). command: ["mysql", "-h", "127.0.0.1", "-e", "SELECT 1"] initialDelaySeconds: 5 periodSeconds: 2 timeoutSeconds: 1 #定义容器 - name: xtrabackup image: harbor.zzhz.com/linux39/xtrabackup:1.0 ports: - name: xtrabackup containerPort: 3307 command: - bash - "-c" - | set -ex cd /var/lib/mysql # Determine binlog position of cloned data, if any. if [[ -f xtrabackup_slave_info ]]; then # XtraBackup already generated a partial "CHANGE MASTER TO" query # because we're cloning from an existing slave. mv xtrabackup_slave_info change_master_to.sql.in # Ignore xtrabackup_binlog_info in this case (it's useless). rm -f xtrabackup_binlog_info elif [[ -f xtrabackup_binlog_info ]]; then # We're cloning directly from master. Parse binlog position. [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1 rm xtrabackup_binlog_info echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\ MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in fi # Check if we need to complete a clone by starting replication. if [[ -f change_master_to.sql.in ]]; then echo "Waiting for mysqld to be ready (accepting connections)" until mysql -h 127.0.0.1 -e "SELECT 1"; do sleep 1; done echo "Initializing replication from clone position" # In case of container restart, attempt this at-most-once. mv change_master_to.sql.in change_master_to.sql.orig mysql -h 127.0.0.1 <<EOF $(<change_master_to.sql.orig), MASTER_HOST='mysql-0.mysql', MASTER_USER='root', MASTER_PASSWORD='', MASTER_CONNECT_RETRY=10; START SLAVE; EOF fi # Start a server to send backups when requested by peers. exec ncat --listen --keep-open --send-only --max-conns=1 3307 -c \ "xtrabackup --backup --slave-info --stream=xbstream --host=127.0.0.1 --user=root" volumeMounts: - name: data #调用PVC模版 mountPath: /var/lib/mysql subPath: mysql - name: conf #调用emptyDir mountPath: /etc/mysql/conf.d resources: requests: cpu: 100m memory: 100Mi volumes: - name: conf #定义和启用emptyDir emptyDir: {} - name: config-map #定义configmap configMap: name: mysql volumeClaimTemplates: #定义PVC模版, - metadata: name: data #PVC模版名称：PVC名称==PVC模版名称+pod名：如下面的data-mysql-0 spec: accessModes: ["ReadWriteOnce"] resources: requests: storage: 10Gi #查看sc，pv，pvc，pod，statefulset，svc [root@localhost7C mysql]# kubectl get pv,pvc,pod,svc,statefulsets -A -o wide NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE VOLUMEMODE persistentvolume/mysql-datadir-1 50Gi RWO Retain Bound default/data-mysql-0 140m Filesystem persistentvolume/mysql-datadir-2 50Gi RWO Retain Bound default/data-mysql-1 140m Filesystem persistentvolume/mysql-datadir-3 50Gi RWO Retain Available 140m Filesystem persistentvolume/mysql-datadir-4 50Gi RWO Retain Bound default/data-mysql-2 140m Filesystem NAMESPACE NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE VOLUMEMODE default persistentvolumeclaim/data-mysql-0 Bound mysql-datadir-1 50Gi RWO 84m Filesystem default persistentvolumeclaim/data-mysql-1 Bound mysql-datadir-2 50Gi RWO 77m Filesystem default persistentvolumeclaim/data-mysql-2 Bound mysql-datadir-4 50Gi RWO 74m Filesystem NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES default pod/mysql-0 2/2 Running 0 30m 10.20.4.54 192.168.80.170 <none> <none> default pod/mysql-1 2/2 Running 1 77m 10.20.5.50 192.168.80.160 <none> <none> NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR default service/mysql ClusterIP None <none> 3306/TCP 138m app=mysql default service/mysql-read ClusterIP 10.10.124.250 <none> 3306/TCP 138m app=mysql NAMESPACE NAME READY AGE CONTAINERS IMAGES default statefulset.apps/mysql 2/2 79m mysql,xtrabackup harbor.zzhz.com/linux39/mysql:5.7,harbor.zzhz.com/linux39/xtrabackup:1.0 #测试短和长域名 FQDN：$(podname).(headless server name) FQDN：$(podname).(headless server name).namespace.svc.zzhz.local [root@localhost7C mysql]# kubectl exec -it mysql-0 bash root@mysql-0:/# hostname mysql-0 root@mysql-0:/# ping mysql-1.mysql.default.svc.zzhz.local root@mysql-0:/# ping mysql-0.mysql.default.svc.zzhz.local root@mysql-0:/# ping mysql-1.mysql