526互联

ETCD集群安装

发布时间 2023-08-24 14:28:51作者: 渣渣中的战斗机

一、证书类型介绍

client certificate 用于通过服务器验证客户端。例如etcdctl，etcd proxy，fleetctl或docker客户端。

server certificate 由服务器使用，并由客户端验证服务器身份。例如docker服务器或kube-apiserver。

peer certificate 由 etcd 集群成员使用，供它们彼此之间通信使用。

二、证书生成

1、cfssl 安装

下载 cfssl，命令如下：

  1. [root@localhost etcd]# wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
  2. [root@localhost etcd]# mv cfssl_linux-amd64 /usr/bin/cfssl

2、cfssljson 安装

  1. [root@localhost etcd]# wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
  2. [root@localhost etcd]# mv cfssljson_linux-amd64 /usr/bin/cfssljson　

3、添加可执行权限

  1. [root@localhost etcd]# chmod +x /usr/bin/{cfssl,cfssljson}

4、配置CA选项

  1. [root@localhost etcd]# mkdir etcd-ca-gen
  2. [root@localhost etcd]# cd etcd-ca-gen
  3. [root@localhost etcd]# cat > ca-config.json << EOF
  4. {
  5. "signing": {
  6. "default": {
  7. "expiry": "43800h"
  8. },
  9. "profiles": {
  10. "server": {
  11. "expiry": "43800h",
  12. "usages": [
  13. "signing",
  14. "key encipherment",
  15. "server auth",
  16. "client auth"
  17. ]
  18. },
  19. "client": {
  20. "expiry": "43800h",
  21. "usages": [
  22. "signing",
  23. "key encipherment",
  24. "client auth"
  25. ]
  26. },
  27. "peer": {
  28. "expiry": "43800h",
  29. "usages": [
  30. "signing",
  31. "key encipherment",
  32. "server auth",
  33. "client auth"
  34. ]
  35. }
  36. }
  37. }
  38. }
  39. EOF
  40. [root@localhost etcd]# cat > ca-csr.json <<EOF
  41. {
  42. "CN": "WAE Etcd CA",
  43. "key": {
  44. "algo": "rsa",
  45. "size": 2048
  46. },
  47. "names": [
  48. {
  49. "C": "CN",
  50. "L": "FJ",
  51. "ST": "Xia Men"
  52. }
  53. ]
  54. }
  55. EOF

5、生成CA证书

  1. [root@localhost etcd]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -#将会生成以下几个文件：
  2. ca-key.pem
  3. ca.csr
  4. ca.pem

6、生成服务器端证书

  1. [root@localhost etcd]# cat > server.json <<EOF
  2. {
  3. "CN": "WAE Etcd Server",
  4. "hosts": [
  5. "etcd1-com-hakim.com",
  6. "etcd2-com-hakim.com",
  7. "etcd3-com-hakim.com",
  8. "etcd4-com-hakim.com",
  9. "etcd5-com-hakim.com"
  10. ],
  11. "key": {
  12. "algo": "rsa",
  13. "size": 2048
  14. },
  15. "names": [
  16. {
  17. "C": "CN",
  18. "L": "FJ",
  19. "ST": "Xia Men"
  20. }
  21. ]
  22. }
  23. EOF
  25. [root@localhost etcd]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server server.json | cfssljson -bare server

7、生成对等证书

  1. $ cp server.json member.json
  2. $ cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer member.json | cfssljson -bare member

8、生成客户端证书

  1. [root@localhost etcd]# cat > client.json <<EOF
  2. {
  3. "CN": "client",
  4. "hosts": [""],
  5. "key": {
  6. "algo": "rsa",
  7. "size": 2048
  8. },
  9. "names": [
  10. {
  11. "C": "CN",
  12. "L": "FJ",
  13. "ST": "Xia Men"
  14. }
  15. ]
  16. }
  17. EOF
  19. [root@localhost etcd]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client.json | cfssljson -bare client

9、保存证书和密钥

将所有 pem 文件复制到 /etc/etcd/etcd-ca/ 目录下，命令如下：

  1. [root@localhost etcd]# mkdir /etc/etcd/etcd-ca/
  2. [root@localhost etcd]# cp *.pem /etc/etcd/etcd-ca/     #注意：把证书拷贝到所有etcd节点，且目录结构相同

三、配置Etcd

1、下载及配置目录

  1. [root@localhost]# tar xvf etcd-v3.4.27-linux-amd64.tar.gz -C /opt
  2. [root@localhost]#  cp etcd etcdctl /usr/bin/
  3. [root@localhost]# chmod 755 /usr/bin/{etcd,etcdctl}
  4. [root@localhost]# mkdir /data/etcd -p         #创建数据目录
  5. [root@localhost]#  mkdir /var/lib/etcd          #创建工作目录

2、创建集群配置文件

第一台配置

[root@localhost]#  cat /etc/etcd/etcd.conf

# [member]
ETCD_NAME=k8s-master01
ETCD_DATA_DIR="/data/etcd/"
ETCD_LISTEN_PEER_URLS="https://192.168.1.87:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.87:2379,https://127.0.0.1:2379"
#[cluster]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.87:2380"
ETCD_INITIAL_CLUSTER="k8s-master01=https://192.168.1.87:2380,k8s-master02=https://192.168.1.85:2380,k8s-master03=https://192.168.1.91:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="smartgo"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.87:2379"
#[security]
ETCD_CERT_FILE="/etc/etcd/etcd-ca/server.pem"
ETCD_KEY_FILE="/etc/etcd/etcd-ca/server-key.pem"
ETCD_CLIENT_CERT_AUTH="true"
ETCD_TRUSTED_CA_FILE="/etc/etcd/etcd-ca/ca.pem"
ETCD_AUTO_TLS="true"
ETCD_PEER_CERT_FILE="/etc/etcd/etcd-ca/member.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/etcd-ca/member-key.pem"
ETCD_PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/etcd-ca/ca.pem"
ETCD_PEER_AUTO_TLS="true"

第二台配置

# [member]
ETCD_NAME=k8s-master02
ETCD_DATA_DIR="/data/etcd/"
ETCD_LISTEN_PEER_URLS="https://192.168.1.85:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.85:2379,https://127.0.0.1:2379"
#[cluster]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.85:2380"
ETCD_INITIAL_CLUSTER="k8s-master01=https://192.168.1.87:2380,k8s-master02=https://192.168.1.85:2380,k8s-master03=https://192.168.1.91:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="smartgo"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.85:2379"
#[security]
ETCD_CERT_FILE="/etc/etcd/etcd-ca/server.pem"
ETCD_KEY_FILE="/etc/etcd/etcd-ca/server-key.pem"
ETCD_CLIENT_CERT_AUTH="true"
ETCD_TRUSTED_CA_FILE="/etc/etcd/etcd-ca/ca.pem"
ETCD_AUTO_TLS="true"
ETCD_PEER_CERT_FILE="/etc/etcd/etcd-ca/member.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/etcd-ca/member-key.pem"
ETCD_PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/etcd-ca/ca.pem"
ETCD_PEER_AUTO_TLS="true"

第三台配置

# [member]
ETCD_NAME=k8s-master03
ETCD_DATA_DIR="/data/etcd/"
ETCD_LISTEN_PEER_URLS="https://192.168.1.91:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.91:2379,https://127.0.0.1:2379"
#[cluster]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.91:2380"
ETCD_INITIAL_CLUSTER="k8s-master01=https://192.168.1.87:2380,k8s-master02=https://192.168.1.85:2380,k8s-master03=https://192.168.1.91:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="smartgo"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.91:2379"
#[security]
ETCD_CERT_FILE="/etc/etcd/etcd-ca/server.pem"
ETCD_KEY_FILE="/etc/etcd/etcd-ca/server-key.pem"
ETCD_CLIENT_CERT_AUTH="true"
ETCD_TRUSTED_CA_FILE="/etc/etcd/etcd-ca/ca.pem"
ETCD_AUTO_TLS="true"
ETCD_PEER_CERT_FILE="/etc/etcd/etcd-ca/member.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/etcd-ca/member-key.pem"
ETCD_PEER_CLIENT_CERT_AUTH="true"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/etcd-ca/ca.pem"
ETCD_PEER_AUTO_TLS="true"

3、制作etcd启动文件

[root@localhost]# cat /usr/lib/systemd/system/etcd.service

[Unit]
Description=Etcd Server
After=network.target
[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
EnvironmentFile=-/etc/etcd/etcd.conf
ExecStart=/usr/bin/etcd
restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

[root@localhost]#systemctl daemon-reload

[root@localhost]#systemctl start etcd

[root@localhost]#systemctl status etcd

部署etcd错误总结

1、conflicting environment variable "ETCD_NAME" is shadowed by corresponding command-line flag (either unset environment variable or disable flag)

原因：ETCD3.4版本会自动读取环境变量的参数，所以EnvironmentFile文件中有的参数，不需要再次在ExecStart启动参数中添加，二选一，如同时配置，会触发以下类似报错

etcd: conflicting environment variable "ETCD_NAME" is shadowed by corresponding command-line flag (either unset environment variable or disable flag)

解决：剔除ExecStart中和配置文件重复的内容即可

2、Failed at step CHDIR spawning /usr/bin/etcd: No such file or directory

原因：etcd.service服务配置文件中设置的工作目录WorkingDirectory=/var/lib/etcd/必须存在，否则会报以上错误

3、http 与 https 之间不能切换，由于刚开始创建时配置文件中写的是http,后修改为https，所以集群一直报错

先通过 http 建立了 cluster，然后再用自签证书 https 来建立，这样就会报错：

error "tls: first record does not look like a TLS handshake"

经过验证：无论是从 http => https，还是从 https => http 的切换都会报这个错，因为一旦建立 cluster 成功，则把连接的协议(http/https) 写入到 etcd 存储里了，不能再更改连接协议。

解决：如果真正遇到需要切换协议，可尝试下面方式

  • 允许删除数据：删除后重新建立 cluster     本文中rm -rf  /data/etcd/*