[root@m4221001 ~]# oc -n openshift-storage logs rook-ceph-osd-prepare-1f7886ec18b9f62db254ab8dcdbfbae6-t47xt
Defaulted container "provision" out of: provision, copy-bins (init), blkdevmapper (init)
2025/07/03 12:11:19 maxprocs: Leaving GOMAXPROCS=16: CPU quota undefined
2025-07-03 12:11:19.170954 I | cephcmd: desired devices to configure osds: [{Name:/mnt/ocs-deviceset-localblock-0-data-27skp4 OSDsPerDevice:1 MetadataDevice: DatabaseSizeMB:0 DeviceClass: InitialWeight: IsFilter:false IsDevicePathFilter:false}]
2025-07-03 12:11:19.178934 I | rookcmd: starting Rook v4.18.7-0.73c94c18237524a3fe266314a9e779ff5c972194 with arguments '/rook/rook ceph osd provision'
2025-07-03 12:11:19.178986 I | rookcmd: flag values: --cluster-id=8afc80c8-9e41-4ebe-b4c8-8221bcca6b96, --cluster-name=ocs-storagecluster-cephcluster, --data-device-filter=, --data-device-path-filter=, --data-devices=[{"id":"/mnt/ocs-deviceset-localblock-0-data-27skp4","storeConfig":{"osdsPerDevice":1}}], --encrypted-device=false, --force-format=false, --help=false, --location=, --log-level=DEBUG, --metadata-device=, --node-name=ocs-deviceset-localblock-0-data-27skp4, --osd-crush-device-class=ssd, --osd-crush-initial-weight=, --osd-database-size=0, --osd-store-type=bluestore, --osd-wal-size=576, --osds-per-device=1, --pvc-backed-osd=true, --replace-osd=1
2025-07-03 12:11:19.179013 I | ceph-spec: parsing mon endpoints: c=172.30.254.248:3300,a=172.30.45.112:3300,b=172.30.176.224:3300
2025-07-03 12:11:19.282686 I | op-osd: CRUSH location=root=default host=worker-2-odf-ci-2-test-ocs
2025-07-03 12:11:19.282751 I | cephcmd: crush location of osd: root=default host=worker-2-odf-ci-2-test-ocs
2025-07-03 12:11:19.348508 I | cephclient: writing config file /var/lib/rook/openshift-storage/openshift-storage.config
2025-07-03 12:11:19.350273 I | cephclient: generated admin config in /var/lib/rook/openshift-storage
2025-07-03 12:11:19.350565 D | cephclient: config file @ /etc/ceph/ceph.conf:
[global]
fsid                               = bc41b8f3-a8df-4b00-92fa-fe9663c60aea
mon initial members                = c a b
mon host                           = [v2:172.30.254.248:3300],[v2:172.30.45.112:3300],[v2:172.30.176.224:3300]
bdev_flock_retry                   = 20
mon_osd_full_ratio                 = .85
mon_osd_backfillfull_ratio         = .8
mon_osd_nearfull_ratio             = .75
mon_max_pg_per_osd                 = 600
mon_pg_warn_max_object_skew        = 0
mon_data_avail_warn                = 15
mon_warn_on_pool_no_redundancy     = false
bluestore_prefer_deferred_size_hdd = 0
bluestore_slow_ops_warn_lifetime   = 0

[osd]
osd_memory_target_cgroup_limit_ratio = 0.8

[client.rbd-mirror.a]
debug_ms         = 1
debug_rbd        = 15
debug_rbd_mirror = 30
log_file         = /var/log/ceph/\$cluster-\$name.log

[client.rbd-mirror-peer]
debug_ms         = 1
debug_rbd        = 15
debug_rbd_mirror = 30
log_file         = /var/log/ceph/\$cluster-\$name.log

[client.admin]
keyring = /var/lib/rook/openshift-storage/client.admin.keyring
2025-07-03 12:11:19.350573 I | cephcmd: destroying osd.1 and cleaning its backing device
2025-07-03 12:11:19.350943 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log lvm list  --format json
2025-07-03 12:11:20.051550 D | cephosd: {}
2025-07-03 12:11:20.051612 I | cephosd: 0 ceph-volume lvm osd devices configured on this node
2025-07-03 12:11:20.051654 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log raw list --format json
2025-07-03 12:11:33.750011 D | cephosd: {
    "e06f36bc-4b63-4dd3-9ab3-52499cb57bea": {
        "ceph_fsid": "bc41b8f3-a8df-4b00-92fa-fe9663c60aea",
        "device": "/dev/sdd",
        "osd_id": 1,
        "osd_uuid": "e06f36bc-4b63-4dd3-9ab3-52499cb57bea",
        "type": "bluestore-rdr"
    }
}
2025-07-03 12:11:33.750527 I | cephosd: 1 ceph-volume raw osd devices configured on this node
2025-07-03 12:11:33.750626 I | cephosd: destroying osd.1
2025-07-03 12:11:33.750674 D | exec: Running command: ceph osd destroy osd.1 --yes-i-really-mean-it --connect-timeout=15 --cluster=openshift-storage --conf=/var/lib/rook/openshift-storage/openshift-storage.config --name=client.admin --keyring=/var/lib/rook/openshift-storage/client.admin.keyring --format json
2025-07-03 12:11:34.292132 I | cephosd: successfully destroyed osd.1
2025-07-03 12:11:34.292297 D | exec: Running command: lsblk /mnt/ocs-deviceset-localblock-0-data-27skp4 --bytes --nodeps --pairs --paths --output SIZE,ROTA,RO,TYPE,PKNAME,NAME,KNAME,MOUNTPOINT,FSTYPE
2025-07-03 12:11:34.295462 D | sys: lsblk output: "SIZE=\"536870912000\" ROTA=\"1\" RO=\"0\" TYPE=\"mpath\" PKNAME=\"\" NAME=\"/dev/mapper/mpathb\" KNAME=\"/dev/dm-3\" MOUNTPOINT=\"\" FSTYPE=\"ceph_bluestore\""
2025-07-03 12:11:34.295550 I | cephosd: zap OSD.1 path "/dev/mapper/mpathb"
2025-07-03 12:11:34.295570 D | exec: Running command: stdbuf -oL ceph-volume lvm zap /dev/mapper/mpathb --destroy
2025-07-03 12:11:35.966375 I | cephosd: --> Zapping: /dev/mapper/mpathb
--> Removing all BlueStore signature on /dev/mapper/mpathb if any...
Running command: /usr/bin/ceph-bluestore-tool zap-device --dev /dev/mapper/mpathb --yes-i-really-really-mean-it
Running command: /usr/bin/dd if=/dev/zero of=/dev/mapper/mpathb bs=1M count=10 conv=fsync
 stderr: 10+0 records in
10+0 records out
 stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0195526 s, 536 MB/s
--> Zapping successful for: <Raw Device: /dev/mapper/mpathb>
2025-07-03 12:11:35.966536 I | cephosd: successfully zapped osd.1 path "/dev/mapper/mpathb"
2025-07-03 12:11:35.966562 D | exec: Running command: dmsetup version
2025-07-03 12:11:35.968596 I | cephosd: Library version:   1.02.202-RHEL9 (2024-11-04)
Driver version:    4.48.0
2025-07-03 12:11:35.984137 I | cephosd: discovering hardware
2025-07-03 12:11:35.984167 D | exec: Running command: lsblk /mnt/ocs-deviceset-localblock-0-data-27skp4 --bytes --nodeps --pairs --paths --output SIZE,ROTA,RO,TYPE,PKNAME,NAME,KNAME,MOUNTPOINT,FSTYPE
2025-07-03 12:11:35.988113 D | sys: lsblk output: "SIZE=\"536870912000\" ROTA=\"1\" RO=\"0\" TYPE=\"mpath\" PKNAME=\"\" NAME=\"/dev/mapper/mpathb\" KNAME=\"/dev/dm-3\" MOUNTPOINT=\"\" FSTYPE=\"\""
2025-07-03 12:11:35.988140 D | exec: Running command: udevadm info --query=property /dev/dm-3
2025-07-03 12:11:35.995462 D | sys: udevadm info output: "DEVPATH=/devices/virtual/block/dm-3\nDEVNAME=/dev/dm-3\nDEVTYPE=disk\nDISKSEQ=8\nMAJOR=253\nMINOR=3\nSUBSYSTEM=block\nUSEC_INITIALIZED=10142359\nDM_UDEV_DISABLE_LIBRARY_FALLBACK_FLAG=1\nDM_UDEV_PRIMARY_SOURCE_FLAG=1\nDM_UDEV_RULES_VSN=2\nDM_NAME=mpathb\nDM_UUID=mpath-360050763808104bc2800000000000076\nDM_SUSPENDED=0\nMPATH_DEVICE_READY=1\nMPATH_SBIN_PATH=/sbin\nDM_TYPE=scsi\nDM_WWN=0x60050763808104bc2800000000000076\nDM_SERIAL=360050763808104bc2800000000000076\nNVME_HOST_IFACE=none\nSYSTEMD_READY=1\nDEVLINKS=/dev/disk/by-id/wwn-0x60050763808104bc2800000000000076 /dev/disk/by-id/dm-uuid-mpath-360050763808104bc2800000000000076 /dev/disk/by-id/scsi-360050763808104bc2800000000000076 /dev/mapper/mpathb /dev/disk/by-id/dm-name-mpathb\nTAGS=:systemd:\nCURRENT_TAGS=:systemd:"
2025-07-03 12:11:35.995535 I | cephosd: creating and starting the osds
2025-07-03 12:11:35.995578 D | cephosd: desiredDevices are [{Name:/mnt/ocs-deviceset-localblock-0-data-27skp4 OSDsPerDevice:1 MetadataDevice: DatabaseSizeMB:0 DeviceClass: InitialWeight: IsFilter:false IsDevicePathFilter:false}]
2025-07-03 12:11:35.995595 D | cephosd: context.Devices are:
2025-07-03 12:11:35.995643 D | cephosd: &{Name:/mnt/ocs-deviceset-localblock-0-data-27skp4 Parent: HasChildren:false DevLinks:/dev/disk/by-id/wwn-0x60050763808104bc2800000000000076 /dev/disk/by-id/dm-uuid-mpath-360050763808104bc2800000000000076 /dev/disk/by-id/scsi-360050763808104bc2800000000000076 /dev/mapper/mpathb /dev/disk/by-id/dm-name-mpathb Size:536870912000 UUID: Serial: Type:data Rotational:true Readonly:false Partitions:[] Filesystem: Mountpoint: Vendor: Model: WWN: WWNVendorExtension: Empty:false CephVolumeData: RealPath:/dev/mapper/mpathb KernelName:dm-3 Encrypted:false}
2025-07-03 12:11:35.995665 I | cephosd: old lsblk can't detect bluestore signature, so try to detect here
2025-07-03 12:11:35.995731 D | exec: Running command: cryptsetup luksDump /mnt/ocs-deviceset-localblock-0-data-27skp4
2025-07-03 12:11:36.005410 E | cephosd: failed to determine if the encrypted block "/mnt/ocs-deviceset-localblock-0-data-27skp4" is from our cluster. failed to dump LUKS header for disk "/mnt/ocs-deviceset-localblock-0-data-27skp4". Device /mnt/ocs-deviceset-localblock-0-data-27skp4 is not a valid LUKS device.: exit status 1
2025-07-03 12:11:36.005545 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log raw list /mnt/ocs-deviceset-localblock-0-data-27skp4 --format json
2025-07-03 12:11:36.395324 D | cephosd: {}
2025-07-03 12:11:36.395487 I | cephosd: 0 ceph-volume raw osd devices configured on this node
2025-07-03 12:11:36.395514 I | cephosd: device "/mnt/ocs-deviceset-localblock-0-data-27skp4" is available.
2025-07-03 12:11:36.395539 I | cephosd: "/mnt/ocs-deviceset-localblock-0-data-27skp4" found in the desired devices
2025-07-03 12:11:36.395574 I | cephosd: device "/mnt/ocs-deviceset-localblock-0-data-27skp4" is selected by the device filter/name "/mnt/ocs-deviceset-localblock-0-data-27skp4"
2025-07-03 12:11:36.406078 I | cephosd: configuring osd devices: {"Entries":{"data":{"Data":-1,"Metadata":null,"Config":{"Name":"/mnt/ocs-deviceset-localblock-0-data-27skp4","OSDsPerDevice":1,"MetadataDevice":"","DatabaseSizeMB":0,"DeviceClass":"ssd","InitialWeight":"","IsFilter":false,"IsDevicePathFilter":false},"PersistentDevicePaths":["/dev/disk/by-id/wwn-0x60050763808104bc2800000000000076","/dev/disk/by-id/dm-uuid-mpath-360050763808104bc2800000000000076","/dev/disk/by-id/scsi-360050763808104bc2800000000000076","/dev/mapper/mpathb","/dev/disk/by-id/dm-name-mpathb"],"DeviceInfo":{"name":"/mnt/ocs-deviceset-localblock-0-data-27skp4","parent":"","hasChildren":false,"devLinks":"/dev/disk/by-id/wwn-0x60050763808104bc2800000000000076 /dev/disk/by-id/dm-uuid-mpath-360050763808104bc2800000000000076 /dev/disk/by-id/scsi-360050763808104bc2800000000000076 /dev/mapper/mpathb /dev/disk/by-id/dm-name-mpathb","size":536870912000,"uuid":"","serial":"","type":"data","rotational":true,"readOnly":false,"Partitions":null,"filesystem":"","mountpoint":"","vendor":"","model":"","wwn":"","wwnVendorExtension":"","empty":false,"real-path":"/dev/mapper/mpathb","kernel-name":"dm-3"},"RestoreOSD":false}}}
2025-07-03 12:11:36.406200 I | cephclient: getting or creating ceph auth key "client.bootstrap-osd"
2025-07-03 12:11:36.406221 D | exec: Running command: ceph auth get-or-create-key client.bootstrap-osd mon allow profile bootstrap-osd --connect-timeout=15 --cluster=openshift-storage --conf=/var/lib/rook/openshift-storage/openshift-storage.config --name=client.admin --keyring=/var/lib/rook/openshift-storage/client.admin.keyring --format json
2025-07-03 12:11:36.969498 D | exec: Running command: lsblk /mnt/ocs-deviceset-localblock-0-data-27skp4 --bytes --nodeps --pairs --paths --output SIZE,ROTA,RO,TYPE,PKNAME,NAME,KNAME,MOUNTPOINT,FSTYPE
2025-07-03 12:11:36.974397 D | sys: lsblk output: "SIZE=\"536870912000\" ROTA=\"1\" RO=\"0\" TYPE=\"mpath\" PKNAME=\"\" NAME=\"/dev/mapper/mpathb\" KNAME=\"/dev/dm-3\" MOUNTPOINT=\"\" FSTYPE=\"\""
2025-07-03 12:11:36.974463 I | cephosd: configuring new device "/mnt/ocs-deviceset-localblock-0-data-27skp4"
2025-07-03 12:11:36.974493 D | exec: Running command: stdbuf -oL ceph-volume --log-path /var/log/ceph/ocs-deviceset-localblock-0-data-27skp4 raw prepare --bluestore --data /mnt/ocs-deviceset-localblock-0-data-27skp4 --osd-id 1 --crush-device-class ssd
2025-07-03 12:11:45.355974 I | cephosd: stderr: Unknown device "/mnt/ocs-deviceset-localblock-0-data-27skp4": No such device
Running command: /usr/bin/ceph-authtool --gen-print-key
Running command: /usr/bin/ceph-authtool --gen-print-key
Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring osd tree -f json
Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 9184df0c-2bae-4b2a-9096-9c8c9caf0afe 1
Running command: /usr/bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-1
Running command: /usr/bin/chown -R ceph:ceph /mnt/ocs-deviceset-localblock-0-data-27skp4
Running command: /usr/bin/ln -s /mnt/ocs-deviceset-localblock-0-data-27skp4 /var/lib/ceph/osd/ceph-1/block
Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-1/activate.monmap
 stderr: got monmap epoch 4
--> Creating keyring file for osd.1
Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1/keyring
Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1/
Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 1 --monmap /var/lib/ceph/osd/ceph-1/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-1/ --osd-uuid 9184df0c-2bae-4b2a-9096-9c8c9caf0afe --setuser ceph --setgroup ceph
 stderr: 2025-07-03T12:11:40.246+0000 3ffb5be1900 -1 bluestore(/var/lib/ceph/osd/ceph-1//block) No valid bdev label found
 stderr: 2025-07-03T12:11:40.596+0000 3ffb5be1900 -1 bluestore(/var/lib/ceph/osd/ceph-1/) _read_fsid unparsable uuid
--> ceph-volume raw dmcrypt prepare successful for: /mnt/ocs-deviceset-localblock-0-data-27skp4
2025-07-03 12:11:45.356225 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log lvm list /mnt/ocs-deviceset-localblock-0-data-27skp4 --format json
2025-07-03 12:11:46.627827 D | cephosd: {}
2025-07-03 12:11:46.627892 I | cephosd: 0 ceph-volume lvm osd devices configured on this node
2025-07-03 12:11:46.627905 D | exec: Running command: cryptsetup luksDump /mnt/ocs-deviceset-localblock-0-data-27skp4
2025-07-03 12:11:46.649529 E | cephosd: failed to determine if the encrypted block "/mnt/ocs-deviceset-localblock-0-data-27skp4" is from our cluster. failed to dump LUKS header for disk "/mnt/ocs-deviceset-localblock-0-data-27skp4". Device /mnt/ocs-deviceset-localblock-0-data-27skp4 is not a valid LUKS device.: exit status 1
2025-07-03 12:11:46.649596 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log raw list /mnt/ocs-deviceset-localblock-0-data-27skp4 --format json
2025-07-03 12:11:47.082934 D | cephosd: {
    "9184df0c-2bae-4b2a-9096-9c8c9caf0afe": {
        "ceph_fsid": "bc41b8f3-a8df-4b00-92fa-fe9663c60aea",
        "device": "/mnt/ocs-deviceset-localblock-0-data-27skp4",
        "osd_id": 1,
        "osd_uuid": "9184df0c-2bae-4b2a-9096-9c8c9caf0afe",
        "type": "bluestore"
    }
}
2025-07-03 12:11:47.083052 D | exec: Running command: lsblk /mnt/ocs-deviceset-localblock-0-data-27skp4 --bytes --nodeps --pairs --paths --output SIZE,ROTA,RO,TYPE,PKNAME,NAME,KNAME,MOUNTPOINT,FSTYPE
2025-07-03 12:11:47.086484 D | sys: lsblk output: "SIZE=\"536870912000\" ROTA=\"1\" RO=\"0\" TYPE=\"mpath\" PKNAME=\"\" NAME=\"/dev/mapper/mpathb\" KNAME=\"/dev/dm-3\" MOUNTPOINT=\"\" FSTYPE=\"\""
2025-07-03 12:11:47.086527 I | cephosd: setting device class "hdd" for device "/mnt/ocs-deviceset-localblock-0-data-27skp4"
2025-07-03 12:11:47.086536 I | cephosd: 1 ceph-volume raw osd devices configured on this node
2025-07-03 12:11:47.086588 I | cephosd: devices = [{ID:1 Cluster:ceph UUID:9184df0c-2bae-4b2a-9096-9c8c9caf0afe DevicePartUUID: DeviceClass:hdd BlockPath:/mnt/ocs-deviceset-localblock-0-data-27skp4 MetadataPath: WalPath: SkipLVRelease:true Location:root=default host=worker-2-odf-ci-2-test-ocs LVBackedPV:false CVMode:raw Store:bluestore TopologyAffinity: Encrypted:false ExportService:false NodeName: PVCName:}]
[root@m4221001 ~]#