版本
[root@control1 ~]# ceph -v ceph version 14.2.22 (ca74598065096e6fcbd8433c8779a2be0c889351) nautilus (stable)
问题
RGW将对象的索引信息存储在 <zone>.rgw.buckets.index pool中，在不分片的情况下，一个对象就对应一个bucket，分片的情况下，多个对象对应同一个bucket，在该对象的omap中存储这bucket的索引信息，如果将这个对象误删除，那么就无法看到该bucket中都有哪些对象，但是对象的真实数据是存储在 <zone>.rgw.buckets.data pool中的，只要data中的数据没有被删除，那么仍然可以正常get到该对象，此时只要object rewrite或者重新向该bucket中上传这些对象就可以自动修复index。
示例
1. 准备测试数据
[root@control1 ~]# s3cmd mb s3://nice Bucket 's3://nice/' created [root@control1 ~]# [root@control1 ~]# [root@control1 ~]# s3cmd ls s3://nice [root@control1 ~]# [root@control1 ~]# s3cmd put pushgateway-1.4.2.linux-amd64.tar.gz s3://nice [root@control1 ~]# s3cmd put node_exporter-1.1.2.linux-amd64.tar.gz s3://nice [root@control1 ~]# [root@control1 ~]# [root@control1 ~]# s3cmd ls s3://nice 2023-02-13 06:12 9246179 s3://nice/node_exporter-1.1.2.linux-amd64.tar.gz 2023-02-13 06:09 9189846 s3://nice/pushgateway-1.4.2.linux-amd64.tar.gz
2. 查看底层数据结构
2.1 找到bucket对应的id
[root@control1 ~]# radosgw-admin bucket stats --bucket nice { "bucket": "nice", "num_shards": 0, "tenant": "", "zonegroup": "cc0580a9-21d5-4377-b64b-fabf1f55fb4e", "placement_rule": "default-placement", "explicit_placement": { "data_pool": "", "data_extra_pool": "", "index_pool": "" }, "id": "8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2", "marker": "8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2", "index_type": "Normal", "owner": "1034c29d7fec46e897aa531f832b6e5c", "ver": "0#3", "master_ver": "0#0", "mtime": "2023-02-13 06:07:37.785853Z", "max_marker": "0#", "usage": { "rgw.main": { "size": 18436025, "size_actual": 18440192, "size_utilized": 18436025, "size_kb": 18004, "size_kb_actual": 18008, "size_kb_utilized": 18004, "num_objects": 2 } }, "bucket_quota": { "enabled": false, "check_on_raw": false, "max_size": -1, "max_size_kb": 0, "max_objects": -1 } }
2.2 在index pool中查看该对象的信息
[root@control1 ~]# rados ls -p cn-1.rgw.buckets.index | grep 8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2 .dir.8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2 [root@control1 ~]# rados -p cn-1.rgw.buckets.index listomapkeys .dir.8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2 node_exporter-1.1.2.linux-amd64.tar.gz pushgateway-1.4.2.linux-amd64.tar.gz [root@control1 ~]# [root@control1 ~]# [root@control1 ~]# rados -p cn-1.rgw.buckets.index listomapvals .dir.8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2 node_exporter-1.1.2.linux-amd64.tar.gz value (293 bytes) : 00000000 08 03 1f 01 00 00 26 00 00 00 6e 6f 64 65 5f 65 |......&...node_e| 00000010 78 70 6f 72 74 65 72 2d 31 2e 31 2e 32 2e 6c 69 |xporter-1.1.2.li| 00000020 6e 75 78 2d 61 6d 64 36 34 2e 74 61 72 2e 67 7a |nux-amd64.tar.gz| 00000030 03 00 00 00 00 00 00 00 01 07 03 91 00 00 00 01 |................| 00000040 e3 15 8d 00 00 00 00 00 ea d4 e9 63 7b eb 01 07 |...........c{...| 00000050 20 00 00 00 36 31 65 32 62 39 36 33 66 36 36 66 | ...61e2b963f66f| 00000060 31 65 30 30 36 34 39 63 38 65 34 64 31 66 34 37 |1e00649c8e4d1f47| 00000070 32 39 65 35 20 00 00 00 31 30 33 34 63 32 39 64 |29e5 ...1034c29d| 00000080 37 66 65 63 34 36 65 38 39 37 61 61 35 33 31 66 |7fec46e897aa531f| 00000090 38 33 32 62 36 65 35 63 05 00 00 00 61 64 6d 69 |832b6e5c....admi| 000000a0 6e 12 00 00 00 61 70 70 6c 69 63 61 74 69 6f 6e |n....application| 000000b0 2f 78 2d 67 7a 69 70 e3 15 8d 00 00 00 00 00 00 |/x-gzip.........| 000000c0 00 00 00 08 00 00 00 53 54 41 4e 44 41 52 44 00 |.......STANDARD.| 000000d0 00 00 00 00 00 00 00 00 01 01 02 00 00 00 0c 03 |................| 000000e0 02 32 00 00 00 38 63 30 36 35 64 31 33 2d 66 35 |.2...8c065d13-f5| 000000f0 66 30 2d 34 39 32 33 2d 39 65 61 30 2d 37 35 31 |f0-4923-9ea0-751| 00000100 38 62 37 37 36 30 34 39 35 2e 34 35 31 39 38 33 |8b7760495.451983| 00000110 37 37 2e 33 35 36 37 00 00 00 00 00 00 00 00 00 |77.3567.........| 00000120 00 00 00 00 00 |.....| 00000125 pushgateway-1.4.2.linux-amd64.tar.gz value (291 bytes) : 00000000 08 03 1d 01 00 00 24 00 00 00 70 75 73 68 67 61 |......$...pushga| 00000010 74 65 77 61 79 2d 31 2e 34 2e 32 2e 6c 69 6e 75 |teway-1.4.2.linu| 00000020 78 2d 61 6d 64 36 34 2e 74 61 72 2e 67 7a 0b 00 |x-amd64.tar.gz..| 00000030 00 00 00 00 00 00 01 07 03 91 00 00 00 01 d6 39 |...............9| 00000040 8c 00 00 00 00 00 0a d4 e9 63 7f a9 85 1b 20 00 |.........c.... .| 00000050 00 00 38 35 63 33 62 39 61 30 30 36 36 37 66 34 |..85c3b9a00667f4| 00000060 38 36 32 64 37 62 38 36 31 36 35 66 63 64 30 34 |862d7b86165fcd04| 00000070 39 33 20 00 00 00 31 30 33 34 63 32 39 64 37 66 |93 ...1034c29d7f| 00000080 65 63 34 36 65 38 39 37 61 61 35 33 31 66 38 33 |ec46e897aa531f83| 00000090 32 62 36 65 35 63 05 00 00 00 61 64 6d 69 6e 12 |2b6e5c....admin.| 000000a0 00 00 00 61 70 70 6c 69 63 61 74 69 6f 6e 2f 78 |...application/x| 000000b0 2d 67 7a 69 70 d6 39 8c 00 00 00 00 00 00 00 00 |-gzip.9.........| 000000c0 00 08 00 00 00 53 54 41 4e 44 41 52 44 00 00 00 |.....STANDARD...| 000000d0 00 00 00 00 00 00 01 01 02 00 00 00 0c 0b 01 32 |...............2| 000000e0 00 00 00 38 63 30 36 35 64 31 33 2d 66 35 66 30 |...8c065d13-f5f0| 000000f0 2d 34 39 32 33 2d 39 65 61 30 2d 37 35 31 38 62 |-4923-9ea0-7518b| 00000100 37 37 36 30 34 39 35 2e 34 35 31 39 38 33 37 37 |7760495.45198377| 00000110 2e 33 35 35 33 00 00 00 00 00 00 00 00 00 00 00 |.3553...........| 00000120 00 00 00 |...| 00000123
可以看到index对象通过omap的方式存储了该bucket跟该bucket中对象的索引关系，以及对象的一些元数据信息，比如修改时间，文件类型等，可以通过下面命令来查看：
[root@control1 ~]# radosgw-admin bucket list --bucket nice [ { "name": "node_exporter-1.1.2.linux-amd64.tar.gz", "instance": "", "ver": { "pool": 12, "epoch": 3 }, "locator": "", "exists": "true", "meta": { "category": 1, "size": 9246179, "mtime": "2023-02-13 06:12:58.117566Z", "etag": "61e2b963f66f1e00649c8e4d1f4729e5", "storage_class": "STANDARD", "owner": "1034c29d7fec46e897aa531f832b6e5c", "owner_display_name": "admin", "content_type": "application/x-gzip", "accounted_size": 9246179, "user_data": "", "appendable": "false" }, "tag": "8c065d13-f5f0-4923-9ea0-7518b7760495.45198377.3567", "flags": 0, "pending_map": [], "versioned_epoch": 0 }, { "name": "pushgateway-1.4.2.linux-amd64.tar.gz", "instance": "", "ver": { "pool": 12, "epoch": 11 }, "locator": "", "exists": "true", "meta": { "category": 1, "size": 9189846, "mtime": "2023-02-13 06:09:14.461744Z", "etag": "85c3b9a00667f4862d7b86165fcd0493", "storage_class": "STANDARD", "owner": "1034c29d7fec46e897aa531f832b6e5c", "owner_display_name": "admin", "content_type": "application/x-gzip", "accounted_size": 9189846, "user_data": "", "appendable": "false" }, "tag": "8c065d13-f5f0-4923-9ea0-7518b7760495.45198377.3553", "flags": 0, "pending_map": [], "versioned_epoch": 0 } ]
2.3 查看data pool中该bucket的数据对象
[root@control1 ~]# rados -p cn-1.rgw.buckets.data ls | grep 8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2 8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2_pushgateway-1.4.2.linux-amd64.tar.gz 8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2_node_exporter-1.1.2.linux-amd64.tar.gz 8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2__shadow_.QjFsBiV-vhyjfGzwpTcBBjO7G6rm5Ld_2 8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2__shadow_.QjFsBiV-vhyjfGzwpTcBBjO7G6rm5Ld_1 8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2__shadow_.QU-DyeqL4u3CQLe3KkKlMsDVqde1wK3_2 8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2__shadow_.QU-DyeqL4u3CQLe3KkKlMsDVqde1wK3_1
可以看到从data中对象的命名上，可以找到index和其中对象的关联关系，即8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2这个索引中有2个tar.gz对象，其他中间带 shadow 关键字的对象是上传的文件太大的情况下，做的数据分片。
3. 误删index对象
[root@control1 ~]# rados -p cn-1.rgw.buckets.index rm .dir.8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2 [root@control1 ~]# [root@control1 ~]# s3cmd ls s3://nice ERROR: S3 error: 404 (NoSuchKey) [root@control1 ~]# [root@control1 ~]# radosgw-admin bucket list --bucket nice ERROR: store->list_objects(): (2) No such file or directory [root@control1 ~]# [root@control1 ~]# s3cmd get s3://nice/pushgateway-1.4.2.linux-amd64.tar.gz download: 's3://nice/pushgateway-1.4.2.linux-amd64.tar.g
可以看到要列出该bucket中有哪些对象的操作，都是失败了的，但是如果知道文件名的话，仍然可以单独下载某个对象，即可以做get操作。
4. 恢复index对象
方法一：rewrite object
[root@control1 ~]# radosgw-admin object rewrite --bucket nice --object node_exporter-1.1.2.linux-amd64.tar.gz [root@control1 ~]# s3cmd ls s3://nice 2023-02-13 06:12 9246179 s3://nice/node_exporter-1.1.2.linux-amd64.tar.gz [root@control1 ~]# [root@control1 ~]# rados -p cn-1.rgw.buckets.index ls | grep dir.8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2 .dir.8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2
方法二：重新上传
[root@control1 ~]# s3cmd put pushgateway-1.4.2.linux-amd64.tar.gz s3://nice upload: 'pushgateway-1.4.2.linux-amd64.tar.gz' -> 's3://nice/pushgateway-1.4.2.linux-amd64.tar.gz' [1 of 1] [root@control1 ~]# rados -p cn-1.rgw.buckets.index ls | grep dir.8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2 .dir.8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2 [root@control1 ~]# rados -p cn-1.rgw.buckets.index listomapkeys .dir.8c065d13-f5f0-4923-9ea0-7518b7760495.45180186.2 pushgateway-1.4.2.linux-amd64.tar.gz [root@control1 ~]# s3cmd ls s3://nice 2023-02-13 06:42 9189846 s3://nice/pushgateway-1.4.2.linux-amd64.tar.gz
方法一比较简单，不需要将该文件下载下来再上传，只需要指定object的名字，重新rewrite一遍就可以了。