事件原因
发现一块16Ti的固态硬盘损坏了,需要在ceph中进行移除
解决思路
| 检查项 | 命令 | 预期结果 |
|---|---|---|
| OSD 已 out | ceph osd tree | grep osd.0 |
down/out |
| 可安全销毁 | ceph osd safe-to-destroy osd.0 |
safe-to-destroy |
| 集群状态 | ceph -s |
HEALTH_OK 或 HEALTH_WARN(仅因 osd.0 down) |
| 确认 hostPath 归属 | cat /var/lib/rook/rook-ceph/<fsid>_<osd-fsid>/whoami |
输出 0 |
| 确认对应物理盘 | lsblk -f / ceph osd metadata osd.0 |
定位到 /dev/nvmeXn1 |
root@ceph-store-01:~# lsblk -f | grep nvme
nvme2n1
nvme0n1 ceph_bluestore
nvme1n1 ceph_bluestore
nvme3n1
├─nvme3n1p1 vfat FAT32 6F9B-15B0 1G 1% /boot/efi
└─nvme3n1p2 ext4 1.0 a8132f1e-5fe3-46b7-8b24-5e21badd93bc 771.7G 7% /
nvme4n1 ceph_bluestore
nvme5n1 ceph_bluestore
nvme6n1 ceph_bluestore
nvme7n1 ceph_bluestore
nvme8n1 ceph_bluestore
nvme9n1 ceph_bluestore
nvme10n1 ceph_bluestore
nvme11n1
nvme12n1 ceph_bluestore
nvme13n1 ceph_bluestore
nvme14n1 ceph_bluestore
root@ceph-store-01:~#
root@ceph-store-01:~#
root@ceph-store-01:~#
root@ceph-store-01:~# lsblk | grep nvme
nvme2n1 259:0 0 0B 0 disk
nvme0n1 259:1 0 14T 0 disk
nvme1n1 259:2 0 14T 0 disk
nvme3n1 259:3 0 894.3G 0 disk
├─nvme3n1p1 259:4 0 1G 0 part /boot/efi
└─nvme3n1p2 259:5 0 893.2G 0 part /
nvme4n1 259:7 0 14T 0 disk
nvme5n1 259:9 0 14T 0 disk
nvme6n1 259:11 0 14T 0 disk
nvme7n1 259:13 0 14T 0 disk
nvme8n1 259:15 0 14T 0 disk
nvme9n1 259:17 0 14T 0 disk
nvme10n1 259:19 0 14T 0 disk
nvme11n1 259:21 0 14T 0 disk
nvme12n1 259:23 0 14T 0 disk
nvme13n1 259:25 0 14T 0 disk
nvme14n1 259:27 0 14T 0 disk
root@ceph-store-01:~#
处理过程
# 开启保护
ceph osd set noout
ceph osd set norebalance
ceph osd set nobackfill
## 限速(NVMe 可调低 sleep,减少业务影响)
ceph config set osd osd_max_backfills 1
ceph config set osd osd_recovery_max_active 1
ceph config set osd osd_recovery_sleep_ssd 0
# ceph 侧清理
ceph osd purge 0 --yes-i-really-mean-it
## 清理验证
ceph osd tree | grep osd.0
ceph auth ls | grep osd.0
# rook 侧
## 备份 cephcluster
kubectl -n rook-ceph get cephcluster -o yaml \
> /tmp/cephcluster-store-prod-$(date +%Y%m%d-%H%M%S).yaml
## 删除 cephcluster 中的对应硬盘
## 正常情况下,Rook 在显式模式下,检测到 CR 中设备被移除后,会自动删除对应的 OSD Deployment。 等待 1-2 分钟。如果无法自动清理,可以手动删除 osd.0 的 deployment
# 节点侧清理(ceph-store-01)
## 1. 确认 hostPath 归属(必须输出 0,也就是osd.id)
cat /var/lib/rook/rook-ceph/5a1b2c3d-0000-4000-8000-000000000000_11111111-2222-3333-4444-555555555555/whoami
## 2. 删目录
rm -rf /var/lib/rook/rook-ceph/5a1b2c3d-0000-4000-8000-000000000000_11111111-2222-3333-4444-555555555555
# 解除保护
ceph osd unset noout
ceph osd unset norebalance
ceph osd unset nobackfill
## 如果有
ceph config rm osd osd_max_backfills
ceph config rm osd osd_recovery_max_active
转载请注明来源,欢迎对文章中的引用来源进行考证,欢迎指出任何有错误或不够清晰的表达。可以在下面评论区评论,也可以邮件至 365433079@qq.com