OpenStack对接Ceph平台
目录
1、创建后端需要的存储池（ceph-01节点操作）
# cinder卷的存储池
root@ceph-01:~# ceph osd pool create volumes 32
# glance存储池
root@ceph-01:~# ceph osd pool create images 32
# 备份存储池
root@ceph-01:~# ceph osd pool create backups 32
# 创建实例存储池
root@ceph-01:~# ceph osd pool create vms 32
# 查看存储池
root@ceph-01:~# ceph osd pool ls
.mgr
volumes
images
backups
vms
root@ceph-01:~# ceph osd pool ls detail
pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode on last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr
pool 2 'volumes' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 24 flags hashpspool stripe_width 0
pool 3 'images' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 27 flags hashpspool stripe_width 0
pool 4 'backups' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 30 flags hashpspool stripe_width 0
pool 5 'vms' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 33 flags hashpspool stripe_width 0
2、创建后端用户
2.1 创建密钥（ceph-01节点操作）
在ceph上创建cinder、glance、cinder-backup、nova用户创建密钥，允许访问使用Ceph存储池
2.1.1 创建用户client.cinder
# 对volumes存储池有rwx权限，对vms存储池有rwx权限，对images池有rx权限
root@ceph-01:~# ceph auth get-or-create client.cinder mon "allow r" osd "allow class-read object_prefix rbd_children,allow rwx pool=volumes,allow rwx pool=vms,allow rx pool=images"
- class-read：x的子集，授予用户调用类读取方法的能力
- object_prefix 通过对象名称前缀。下例将访问限制为任何池中名称仅以 rbd_children 为开头的对象。
2.1.2 创建用户client.glance
# 对images存储池有rwx权限
root@ceph-01:~# ceph auth get-or-create client.glance mon "allow r" osd "allow class-read object_prefix rbd_children,allow rwx pool=images"
2.1.3 创建用户client.cinder-backup
# 对backups存储池有rwx权限
root@ceph-01:~# ceph auth get-or-create client.cinder-backup mon "profile rbd" osd "profile rbd pool=backups"
使用 rbd profile 为新的 cinder-backup 用户帐户定义访问权限。然后，客户端应用使用这一帐户基于块来访问利用了 RADOS 块设备的 Ceph 存储。
2.2 导出密钥（ceph-01节点操作）
# 切换到ceph目录
root@ceph-01:~# cd /etc/ceph/
# 导出glance密钥
root@ceph-01:/etc/ceph# ceph auth get client.glance -o ceph.client.glance.keyring
# 导出cinder密钥
root@ceph-01:/etc/ceph# ceph auth get client.cinder -o ceph.client.cinder.keyring
# 导出cinder-backup密钥
root@ceph-01:/etc/ceph# ceph auth get client.cinder-backup -o ceph.client.cinder-backup.keyring
2.3 拷贝密钥
2.3.1 创建存放目录（openstack三节点执行）
root@controller:~# mkdir /etc/ceph/
2.3.2 拷贝密钥到控制节点（ceph-01节点操作）
# 拷贝glance密钥
root@ceph-01:/etc/ceph# scp ceph.client.glance.keyring root@192.168.200.30:/etc/ceph/
# 拷贝cinder密钥
root@ceph-01:/etc/ceph# scp ceph.client.cinder.keyring root@192.168.200.30:/etc/ceph/
# 拷贝ceph集群认证配置文件
root@ceph-01:/etc/ceph# scp ceph.conf root@192.168.200.30:/etc/ceph/
2.3.3 拷贝密钥到计算节点（ceph-01节点操作）
# 拷贝cinder密钥
root@ceph-01:/etc/ceph# scp ceph.client.cinder.keyring root@192.168.200.31:/etc/ceph/
root@ceph-01:/etc/ceph# scp ceph.client.cinder.keyring root@192.168.200.32:/etc/ceph/
# 拷贝cinder-backup密钥（backup服务节点）
root@ceph-01:/etc/ceph# scp ceph.client.cinder-backup.keyring root@192.168.200.31:/etc/ceph/
root@ceph-01:/etc/ceph# scp ceph.client.cinder-backup.keyring root@192.168.200.32:/etc/ceph/
# 拷贝ceph集群认证配置文件
root@ceph-01:/etc/ceph# scp ceph.conf root@192.168.200.31:/etc/ceph/
root@ceph-01:/etc/ceph# scp ceph.conf root@192.168.200.32:/etc/ceph/
3、添加libvirt密钥（计算节点执行）
# 生成密钥（PS：注意，如果有多个计算节点，它们的UUID必须一致）
root@compute-01:~# cd /etc/ceph/
root@compute-01:/etc/ceph# UUID=bf168fa8-8d5b-4991-ba4c-12ae622a98b1
root@compute-01:/etc/ceph# cat >> secret.xml << EOF
<secret ephemeral='no' private='no'>
<uuid>$UUID</uuid>
<usage type='ceph'>
<name>client.cinder secret</name>
</usage>
</secret>
EOF
# 执行命令写入secret
[root@compute01 ~]# virsh secret-define --file secret.xml
# 查看添加后端密钥
root@compute-01:/etc/ceph# virsh secret-list
UUID Usage
-------------------------------------------------------------------
bf168fa8-8d5b-4991-ba4c-12ae622a98b1 ceph client.cinder secret
# 加入key
# 将key值复制出来
[root@compute01 ~]# cat ceph.client.cinder.keyring
AQCvztRk8ssALhAAXshR1E+Y90HvIyxkhal1cQ==
[root@compute01 ~]# virsh secret-set-value --secret ${UUID} --base64 $(cat ceph.client.cinder.keyring | grep key | awk -F ' ' '{print $3}')
4、安装ceph客户端（openstack三节点执行）
# 主要作用是OpenStack可调用Ceph资源
root@controller:~# apt install -y ceph-common
5、配置glance后端存储
5.1 配置glance后端存储（控制节点执行）
# 更改glance密钥属性
root@controller:~# chown glance.glance /etc/ceph/ceph.client.glance.keyring
# 修改配置文件
root@controller:~# vim /etc/glance/glance-api.conf
root@controller:~# sed -n "14,23p" /etc/glance/glance-api.conf
[glance_store]
# stores = file,http
# default_store = file
# filesystem_store_datadir = /var/lib/glance/images/
stores = rbd,file,http
default_store = rbd
rbd_store_pool = images
rbd_store_user = glance
rbd_store_ceph_conf = /etc/ceph/ceph.conf
rbd_store_chunk_size = 8
# 安装缺失aws的模块
root@controller:~# apt install -y python3-boto3
# 重启生效ceph配置
root@controller:~# systemctl restart glance-api
# 设置开机自启动
root@controller:~# systemctl enable glance-api
# 上传镜像
root@controller:~# openstack image create cirros_v1 --disk-format qcow2 --file cirros-0.4.0-x86_64-disk.img
5.2 验证镜像（ceph-01节点执行）
root@ceph-01:/etc/ceph# rbd ls images
6、配置cinder后端存储
6.1 更改cinder密钥属性（openstack三节点执行）
root@controller:~# chown cinder.cinder /etc/ceph/ceph.client.cinder.keyring
6.2 修改cinder配置文件
控制节点执行
# 指定存储类型，否则在创建卷时，类型为 __DEFAULT__
root@controller:~# vim /etc/cinder/cinder.conf
root@controller:~# sed -n "1,2p" /etc/cinder/cinder.conf
[DEFAULT]
default_volume_type = ceph
# 重启服务生效配置
root@controller:~# systemctl restart cinder-scheduler
计算节点执行
root@controller:~# vim /etc/cinder/cinder.conf
root@compute-01:/etc/ceph# sed -n "1,2p;28,39p" /etc/cinder/cinder.conf
[DEFAULT]
enabled_backends = ceph,lvm
[ceph]
volume_driver = cinder.volume.drivers.rbd.RBDDriver
rbd_pool = volumes
rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_flatten_volume_from_snapshot = false
rbd_max_clone_depth = 5
rbd_store_chunk_size = 4
rados_connect_timeout = -1
glance_api_version = 2
rbd_user = cinder
rbd_secret_uuid = bf168fa8-8d5b-4991-ba4c-12ae622a98b1
volume_backend_name = ceph
# 重启服务生效配置
root@compute-01:/etc/ceph# systemctl restart cinder-volume
6.3 创建卷类型（控制节点执行）
# 创建卷类型
root@controller:~# openstack volume type create ceph
# 设置卷类型元数据
root@controller:~# cinder --os-username admin --os-tenant-name admin type-key ceph set volume_backend_name=ceph
# 查看存储类型
root@controller:~# openstack volume type list
+--------------------------------------+-------------+-----------+
| ID | Name | Is Public |
+--------------------------------------+-------------+-----------+
| ccb0cd1e-562a-42e0-b0d0-3818d6910528 | ceph | True |
| 5ffdbebe-4f37-4690-8b40-36c6e6c63233 | lvm | True |
| c5b71526-643d-4e9c-b0b7-3cdf8d1e926b | __DEFAULT__ | True |
+--------------------------------------+-------------+-----------+
# 创建卷测试
root@controller:~# openstack volume create ceph01 --type ceph --size 1
# 查看volumes存储池是否存在卷
root@ceph-01:/etc/ceph# rbd ls volumes
7、卷备份
7.1 配置卷备份（计算节点执行）
# 安装服务
root@compute-01:/etc/ceph# apt -y install cinder-backup
# 更改密钥属性
root@compute-01:/etc/ceph# chown cinder.cinder /etc/ceph/ceph.client.cinder-backup.keyring
# 修改配置文件
root@compute-01:/etc/ceph# vim /etc/cinder/cinder.conf
root@compute-01:/etc/ceph# sed -n "1,9p" /etc/cinder/cinder.conf
[DEFAULT]
backup_driver = cinder.backup.drivers.ceph.CephBackupDriver
backup_ceph_conf=/etc/ceph/ceph.conf
backup_ceph_user = cinder-backup
backup_ceph_chunk_size = 4194304
backup_ceph_pool = backups
backup_ceph_stripe_unit = 0
backup_ceph_stripe_count = 0
restore_discard_excess_bytes = true
# 重启生效配置
root@compute-01:/etc/ceph# systemctl restart cinder-backup
# 设置开机自启动
root@compute-01:/etc/ceph# systemctl enable cinder-backup
7.2 创建卷备份（控制节点执行）
root@controller:~# openstack volume backup create --name ceph_backup ceph01
7.3 验证卷备份（ceph-01节点执行）
root@ceph-01:/etc/ceph# rbd ls backups
8、配置nova集成ceph
8.1 修改nova配置文件（计算节点执行）
# 修改nova配置文件
root@compute-01:/etc/ceph# vim /etc/nova/nova.conf
root@compute-01:/etc/ceph# sed -n "1,2p;47,52p" /etc/nova/nova.conf
[DEFAULT]
live_migration_flag = "VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE"
[libvirt]
images_type = rbd
images_rbd_pool = vms
images_rbd_ceph_conf = /etc/ceph/ceph.conf
rbd_user = cinder
rbd_secret_uuid = bf168fa8-8d5b-4991-ba4c-12ae622a98b1
# 安装qemu支持rbd
root@compute-01:/etc/ceph# apt install -y qemu-block-extra
# 重启nova服务生效配置
root@compute-01:/etc/ceph# systemctl restart nova-compute
8.2 创建实例测试（控制节点执行）
# 安全组对应admin项目ID
openstack server create --flavor C1-512MB-1G --image cirros_v1 --security-group default --nic net-id=$(vxlan网络id) --key-name mykey vm02
root@controller:/etc/ceph# openstack server create --flavor C1-512MB-1G --image cirros --security-group default --nic net-id=a371c907-a326-4869-ab35-8ea53d838a98 --key-name mykey vm03
8.3 验证是否到ceph中的vms存储池（ceph-01节点执行）
root@ceph-01:/etc/ceph# rbd ls vms
0e1edb02-ec96-46f3-8e03-0599f06a1564_disk
8.4 热迁移配置（计算节点执行）
# 备份配置文件
root@compute-01:/etc/ceph# cp /etc/libvirt/libvirtd.conf{,.bak}
# 配置监听地址
root@compute-01:/etc/ceph# vim /etc/libvirt/libvirtd.conf
root@compute-01:/etc/ceph# cat /etc/libvirt/libvirtd.conf
listen_tls = 0
listen_tcp = 1
tcp_port = "16509"
listen_addr = "192.168.200.32" # 注意自己的主机地址
auth_tcp = "none"
# 备份配置文件
root@compute-01:/etc/ceph# cp /etc/default/libvirtd{,.bak}
# 开启监听地址
root@compute-01:/etc/ceph# vim /etc/default/libvirtd
root@compute-01:/etc/ceph# sed -n "3p" /etc/default/libvirtd
LIBVIRTD_ARGS="--listen"
# 屏蔽libvirtd服务
root@compute-01:/etc/ceph# systemctl mask libvirtd.socket libvirtd-ro.socket libvirtd-admin.socket libvirtd-tls.socket libvirtd-tcp.socket
# 重启libvirtd生效配置
root@compute-01:/etc/ceph# systemctl restart libvirtd
# 重启计算节点nova服务
root@compute-01:/etc/ceph# systemctl restart nova-compute
8.5 测试是否能互相通信连接，互通测试再进行热迁移
# compute-01连接compute-02
root@compute-01:/etc/ceph# virsh -c qemu+tcp://compute-02/system
Welcome to virsh, the virtualization interactive terminal.
Type: 'help' for help with commands
'quit' to quit
virsh #
# compute-02连接compute-01
root@compute-02:/etc/ceph# virsh -c qemu+tcp://compute-01/system
Welcome to virsh, the virtualization interactive terminal.
Type: 'help' for help with commands
'quit' to quit
virsh #
8.6 查看云主机
# 查看云主机
root@controller:/etc/ceph# openstack server list --all
+--------------------------------------+------+--------+---------------------------------------+--------------------------+-------------+
| ID | Name | Status | Networks | Image | Flavor |
+--------------------------------------+------+--------+---------------------------------------+--------------------------+-------------+
| 0e1edb02-ec96-46f3-8e03-0599f06a1564 | vm03 | ACTIVE | Intnal=192.168.100.94 | cirros | C1-512MB-1G |
| 5db90dbb-9a25-46da-8d3a-24334c540d1f | vm02 | ACTIVE | Intnal=192.168.100.202 | N/A (booted from volume) | C1-512MB-1G |
| 77705e1a-8dfb-4b07-886b-c773b4b9be4a | vm01 | ACTIVE | Intnal=192.168.100.81, 192.168.200.59 | cirros | C1-512MB-1G |
+--------------------------------------+------+--------+---------------------------------------+--------------------------+-------------+
# 查看需要迁移的云主机详细信息
root@controller:/etc/ceph# openstack server show 0e1edb02-ec96-46f3-8e03-0599f06a1564
+-------------------------------------+----------------------------------------------------------+
| Field | Value |
+-------------------------------------+----------------------------------------------------------+
| OS-DCF:diskConfig | MANUAL |
| OS-EXT-AZ:availability_zone | nova |
| OS-EXT-SRV-ATTR:host | compute-01 |
| OS-EXT-SRV-ATTR:hypervisor_hostname | compute-01 |
| OS-EXT-SRV-ATTR:instance_name | instance-00000003 |
| OS-EXT-STS:power_state | Running |
| OS-EXT-STS:task_state | None |
| OS-EXT-STS:vm_state | active |
| OS-SRV-USG:launched_at | 2023-08-10T13:29:39.000000 |
| OS-SRV-USG:terminated_at | None |
| accessIPv4 | |
| accessIPv6 | |
| addresses | Intnal=192.168.100.94 |
| config_drive | |
| created | 2023-08-10T13:29:32Z |
| flavor | C1-512MB-1G (bcdee760-b585-4455-8a06-825db740e988) |
| hostId | 993f86bcdaeeaac9c515a989902788bf8c3050973d4a5c07a30f97fb |
| id | 0e1edb02-ec96-46f3-8e03-0599f06a1564 |
| image | cirros (ca7a8fff-7296-4907-894d-a84825955ad2) |
| key_name | mykey |
| name | vm03 |
| progress | 0 |
| project_id | d511671fdcbc408fbe1978da791caa46 |
| properties | |
| security_groups | name='default' |
| status | ACTIVE |
| updated | 2023-08-10T13:29:39Z |
| user_id | 3a66f914c2804018a416255998dda997 |
| volumes_attached | |
+-------------------------------------+----------------------------------------------------------+
# 热迁移到另一个计算节点
root@controller:/etc/ceph# nova live-migration 0e1edb02-ec96-46f3-8e03-0599f06a1564 compute-02