[root@m4221001 ~]# oc -n openshift-storage logs rook-ceph-osd-prepare-bd4d27be1b945642c1f2f6a8d9ff051b-pbl5w
Defaulted container "provision" out of: provision, copy-bins (init), blkdevmapper (init)
2025/07/03 12:13:12 maxprocs: Leaving GOMAXPROCS=16: CPU quota undefined
2025-07-03 12:13:12.763225 I | cephcmd: desired devices to configure osds: [{Name:/mnt/ocs-deviceset-localblock-0-data-0wgjtv OSDsPerDevice:1 MetadataDevice: DatabaseSizeMB:0 DeviceClass: InitialWeight: IsFilter:false IsDevicePathFilter:false}]
2025-07-03 12:13:12.764592 I | rookcmd: starting Rook v4.18.7-0.73c94c18237524a3fe266314a9e779ff5c972194 with arguments '/rook/rook ceph osd provision'
2025-07-03 12:13:12.764606 I | rookcmd: flag values: --cluster-id=8afc80c8-9e41-4ebe-b4c8-8221bcca6b96, --cluster-name=ocs-storagecluster-cephcluster, --data-device-filter=, --data-device-path-filter=, --data-devices=[{"id":"/mnt/ocs-deviceset-localblock-0-data-0wgjtv","storeConfig":{"osdsPerDevice":1}}], --encrypted-device=false, --force-format=false, --help=false, --location=, --log-level=DEBUG, --metadata-device=, --node-name=ocs-deviceset-localblock-0-data-0wgjtv, --osd-crush-device-class=ssd, --osd-crush-initial-weight=, --osd-database-size=0, --osd-store-type=bluestore, --osd-wal-size=576, --osds-per-device=1, --pvc-backed-osd=true, --replace-osd=2
2025-07-03 12:13:12.764619 I | ceph-spec: parsing mon endpoints: c=172.30.254.248:3300,a=172.30.45.112:3300,b=172.30.176.224:3300
2025-07-03 12:13:12.784325 I | op-osd: CRUSH location=root=default host=worker-0-odf-ci-2-test-ocs
2025-07-03 12:13:12.784353 I | cephcmd: crush location of osd: root=default host=worker-0-odf-ci-2-test-ocs
2025-07-03 12:13:12.793129 I | cephclient: writing config file /var/lib/rook/openshift-storage/openshift-storage.config
2025-07-03 12:13:12.793594 I | cephclient: generated admin config in /var/lib/rook/openshift-storage
2025-07-03 12:13:12.793824 D | cephclient: config file @ /etc/ceph/ceph.conf:
[global]
fsid                               = bc41b8f3-a8df-4b00-92fa-fe9663c60aea
mon initial members                = c a b
mon host                           = [v2:172.30.254.248:3300],[v2:172.30.45.112:3300],[v2:172.30.176.224:3300]
bdev_flock_retry                   = 20
mon_osd_full_ratio                 = .85
mon_osd_backfillfull_ratio         = .8
mon_osd_nearfull_ratio             = .75
mon_max_pg_per_osd                 = 600
mon_pg_warn_max_object_skew        = 0
mon_data_avail_warn                = 15
mon_warn_on_pool_no_redundancy     = false
bluestore_prefer_deferred_size_hdd = 0
bluestore_slow_ops_warn_lifetime   = 0

[osd]
osd_memory_target_cgroup_limit_ratio = 0.8

[client.rbd-mirror.a]
debug_ms         = 1
debug_rbd        = 15
debug_rbd_mirror = 30
log_file         = /var/log/ceph/\$cluster-\$name.log

[client.rbd-mirror-peer]
debug_ms         = 1
debug_rbd        = 15
debug_rbd_mirror = 30
log_file         = /var/log/ceph/\$cluster-\$name.log

[client.admin]
keyring = /var/lib/rook/openshift-storage/client.admin.keyring
2025-07-03 12:13:12.793836 I | cephcmd: destroying osd.2 and cleaning its backing device
2025-07-03 12:13:12.794094 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log lvm list  --format json
2025-07-03 12:13:13.482782 D | cephosd: {}
2025-07-03 12:13:13.482874 I | cephosd: 0 ceph-volume lvm osd devices configured on this node
2025-07-03 12:13:13.482955 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log raw list --format json
2025-07-03 12:13:26.333902 D | cephosd: {
    "f2861c67-83f0-408c-9154-b5bf6a74b9d0": {
        "ceph_fsid": "bc41b8f3-a8df-4b00-92fa-fe9663c60aea",
        "device": "/dev/sdd",
        "osd_id": 2,
        "osd_uuid": "f2861c67-83f0-408c-9154-b5bf6a74b9d0",
        "type": "bluestore-rdr"
    }
}
2025-07-03 12:13:26.334066 I | cephosd: 1 ceph-volume raw osd devices configured on this node
2025-07-03 12:13:26.334073 I | cephosd: destroying osd.2
2025-07-03 12:13:26.334097 D | exec: Running command: ceph osd destroy osd.2 --yes-i-really-mean-it --connect-timeout=15 --cluster=openshift-storage --conf=/var/lib/rook/openshift-storage/openshift-storage.config --name=client.admin --keyring=/var/lib/rook/openshift-storage/client.admin.keyring --format json
2025-07-03 12:13:26.853580 I | cephosd: successfully destroyed osd.2
2025-07-03 12:13:26.853639 D | exec: Running command: lsblk /mnt/ocs-deviceset-localblock-0-data-0wgjtv --bytes --nodeps --pairs --paths --output SIZE,ROTA,RO,TYPE,PKNAME,NAME,KNAME,MOUNTPOINT,FSTYPE
2025-07-03 12:13:26.856886 D | sys: lsblk output: "SIZE=\"536870912000\" ROTA=\"1\" RO=\"0\" TYPE=\"mpath\" PKNAME=\"\" NAME=\"/dev/mapper/mpathb\" KNAME=\"/dev/dm-3\" MOUNTPOINT=\"\" FSTYPE=\"ceph_bluestore\""
2025-07-03 12:13:26.856913 I | cephosd: zap OSD.2 path "/dev/mapper/mpathb"
2025-07-03 12:13:26.856919 D | exec: Running command: stdbuf -oL ceph-volume lvm zap /dev/mapper/mpathb --destroy
2025-07-03 12:13:28.539819 I | cephosd: --> Zapping: /dev/mapper/mpathb
--> Removing all BlueStore signature on /dev/mapper/mpathb if any...
Running command: /usr/bin/ceph-bluestore-tool zap-device --dev /dev/mapper/mpathb --yes-i-really-really-mean-it
Running command: /usr/bin/dd if=/dev/zero of=/dev/mapper/mpathb bs=1M count=10 conv=fsync
--> Zapping successful for: <Raw Device: /dev/mapper/mpathb>
2025-07-03 12:13:28.539861 I | cephosd: successfully zapped osd.2 path "/dev/mapper/mpathb"
2025-07-03 12:13:28.539877 D | exec: Running command: dmsetup version
2025-07-03 12:13:28.542015 I | cephosd: Library version:   1.02.202-RHEL9 (2024-11-04)
Driver version:    4.48.0
2025-07-03 12:13:28.555332 I | cephosd: discovering hardware
2025-07-03 12:13:28.555366 D | exec: Running command: lsblk /mnt/ocs-deviceset-localblock-0-data-0wgjtv --bytes --nodeps --pairs --paths --output SIZE,ROTA,RO,TYPE,PKNAME,NAME,KNAME,MOUNTPOINT,FSTYPE
2025-07-03 12:13:28.559391 D | sys: lsblk output: "SIZE=\"536870912000\" ROTA=\"1\" RO=\"0\" TYPE=\"mpath\" PKNAME=\"\" NAME=\"/dev/mapper/mpathb\" KNAME=\"/dev/dm-3\" MOUNTPOINT=\"\" FSTYPE=\"\""
2025-07-03 12:13:28.559428 D | exec: Running command: udevadm info --query=property /dev/dm-3
2025-07-03 12:13:28.566660 D | sys: udevadm info output: "DEVPATH=/devices/virtual/block/dm-3\nDEVNAME=/dev/dm-3\nDEVTYPE=disk\nDISKSEQ=8\nMAJOR=253\nMINOR=3\nSUBSYSTEM=block\nUSEC_INITIALIZED=13107186\nDM_UDEV_DISABLE_LIBRARY_FALLBACK_FLAG=1\nDM_UDEV_PRIMARY_SOURCE_FLAG=1\nDM_UDEV_RULES_VSN=2\nDM_NAME=mpathb\nDM_UUID=mpath-360050763808104bc2800000000000074\nDM_SUSPENDED=0\nMPATH_DEVICE_READY=1\nMPATH_SBIN_PATH=/sbin\nDM_TYPE=scsi\nDM_WWN=0x60050763808104bc2800000000000074\nDM_SERIAL=360050763808104bc2800000000000074\nNVME_HOST_IFACE=none\nSYSTEMD_READY=1\nDEVLINKS=/dev/disk/by-id/scsi-360050763808104bc2800000000000074 /dev/mapper/mpathb /dev/disk/by-id/wwn-0x60050763808104bc2800000000000074 /dev/disk/by-id/dm-uuid-mpath-360050763808104bc2800000000000074 /dev/disk/by-id/dm-name-mpathb\nTAGS=:systemd:\nCURRENT_TAGS=:systemd:"
2025-07-03 12:13:28.566683 I | cephosd: creating and starting the osds
2025-07-03 12:13:28.566706 D | cephosd: desiredDevices are [{Name:/mnt/ocs-deviceset-localblock-0-data-0wgjtv OSDsPerDevice:1 MetadataDevice: DatabaseSizeMB:0 DeviceClass: InitialWeight: IsFilter:false IsDevicePathFilter:false}]
2025-07-03 12:13:28.566709 D | cephosd: context.Devices are:
2025-07-03 12:13:28.566732 D | cephosd: &{Name:/mnt/ocs-deviceset-localblock-0-data-0wgjtv Parent: HasChildren:false DevLinks:/dev/disk/by-id/scsi-360050763808104bc2800000000000074 /dev/mapper/mpathb /dev/disk/by-id/wwn-0x60050763808104bc2800000000000074 /dev/disk/by-id/dm-uuid-mpath-360050763808104bc2800000000000074 /dev/disk/by-id/dm-name-mpathb Size:536870912000 UUID: Serial: Type:data Rotational:true Readonly:false Partitions:[] Filesystem: Mountpoint: Vendor: Model: WWN: WWNVendorExtension: Empty:false CephVolumeData: RealPath:/dev/mapper/mpathb KernelName:dm-3 Encrypted:false}
2025-07-03 12:13:28.566738 I | cephosd: old lsblk can't detect bluestore signature, so try to detect here
2025-07-03 12:13:28.566777 D | exec: Running command: cryptsetup luksDump /mnt/ocs-deviceset-localblock-0-data-0wgjtv
2025-07-03 12:13:28.575668 E | cephosd: failed to determine if the encrypted block "/mnt/ocs-deviceset-localblock-0-data-0wgjtv" is from our cluster. failed to dump LUKS header for disk "/mnt/ocs-deviceset-localblock-0-data-0wgjtv". Device /mnt/ocs-deviceset-localblock-0-data-0wgjtv is not a valid LUKS device.: exit status 1
2025-07-03 12:13:28.575698 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log raw list /mnt/ocs-deviceset-localblock-0-data-0wgjtv --format json
2025-07-03 12:13:28.917837 D | cephosd: {}
2025-07-03 12:13:28.917903 I | cephosd: 0 ceph-volume raw osd devices configured on this node
2025-07-03 12:13:28.917913 I | cephosd: device "/mnt/ocs-deviceset-localblock-0-data-0wgjtv" is available.
2025-07-03 12:13:28.917919 I | cephosd: "/mnt/ocs-deviceset-localblock-0-data-0wgjtv" found in the desired devices
2025-07-03 12:13:28.917930 I | cephosd: device "/mnt/ocs-deviceset-localblock-0-data-0wgjtv" is selected by the device filter/name "/mnt/ocs-deviceset-localblock-0-data-0wgjtv"
2025-07-03 12:13:28.938025 I | cephosd: configuring osd devices: {"Entries":{"data":{"Data":-1,"Metadata":null,"Config":{"Name":"/mnt/ocs-deviceset-localblock-0-data-0wgjtv","OSDsPerDevice":1,"MetadataDevice":"","DatabaseSizeMB":0,"DeviceClass":"ssd","InitialWeight":"","IsFilter":false,"IsDevicePathFilter":false},"PersistentDevicePaths":["/dev/disk/by-id/scsi-360050763808104bc2800000000000074","/dev/mapper/mpathb","/dev/disk/by-id/wwn-0x60050763808104bc2800000000000074","/dev/disk/by-id/dm-uuid-mpath-360050763808104bc2800000000000074","/dev/disk/by-id/dm-name-mpathb"],"DeviceInfo":{"name":"/mnt/ocs-deviceset-localblock-0-data-0wgjtv","parent":"","hasChildren":false,"devLinks":"/dev/disk/by-id/scsi-360050763808104bc2800000000000074 /dev/mapper/mpathb /dev/disk/by-id/wwn-0x60050763808104bc2800000000000074 /dev/disk/by-id/dm-uuid-mpath-360050763808104bc2800000000000074 /dev/disk/by-id/dm-name-mpathb","size":536870912000,"uuid":"","serial":"","type":"data","rotational":true,"readOnly":false,"Partitions":null,"filesystem":"","mountpoint":"","vendor":"","model":"","wwn":"","wwnVendorExtension":"","empty":false,"real-path":"/dev/mapper/mpathb","kernel-name":"dm-3"},"RestoreOSD":false}}}
2025-07-03 12:13:28.938159 I | cephclient: getting or creating ceph auth key "client.bootstrap-osd"
2025-07-03 12:13:28.938181 D | exec: Running command: ceph auth get-or-create-key client.bootstrap-osd mon allow profile bootstrap-osd --connect-timeout=15 --cluster=openshift-storage --conf=/var/lib/rook/openshift-storage/openshift-storage.config --name=client.admin --keyring=/var/lib/rook/openshift-storage/client.admin.keyring --format json
2025-07-03 12:13:29.625437 D | exec: Running command: lsblk /mnt/ocs-deviceset-localblock-0-data-0wgjtv --bytes --nodeps --pairs --paths --output SIZE,ROTA,RO,TYPE,PKNAME,NAME,KNAME,MOUNTPOINT,FSTYPE
2025-07-03 12:13:29.630392 D | sys: lsblk output: "SIZE=\"536870912000\" ROTA=\"1\" RO=\"0\" TYPE=\"mpath\" PKNAME=\"\" NAME=\"/dev/mapper/mpathb\" KNAME=\"/dev/dm-3\" MOUNTPOINT=\"\" FSTYPE=\"\""
2025-07-03 12:13:29.630447 I | cephosd: configuring new device "/mnt/ocs-deviceset-localblock-0-data-0wgjtv"
2025-07-03 12:13:29.630475 D | exec: Running command: stdbuf -oL ceph-volume --log-path /var/log/ceph/ocs-deviceset-localblock-0-data-0wgjtv raw prepare --bluestore --data /mnt/ocs-deviceset-localblock-0-data-0wgjtv --osd-id 2 --crush-device-class ssd
2025-07-03 12:13:39.262607 I | cephosd: stderr: Unknown device "/mnt/ocs-deviceset-localblock-0-data-0wgjtv": No such device
Running command: /usr/bin/ceph-authtool --gen-print-key
Running command: /usr/bin/ceph-authtool --gen-print-key
Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring osd tree -f json
Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 5fc7f603-a991-457e-876b-edcf7dd335a9 2
Running command: /usr/bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-2
Running command: /usr/bin/chown -R ceph:ceph /mnt/ocs-deviceset-localblock-0-data-0wgjtv
Running command: /usr/bin/ln -s /mnt/ocs-deviceset-localblock-0-data-0wgjtv /var/lib/ceph/osd/ceph-2/block
Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-2/activate.monmap
 stderr: got monmap epoch 4
--> Creating keyring file for osd.2
Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2/keyring
Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2/
Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 2 --monmap /var/lib/ceph/osd/ceph-2/activate.monmap --keyfile - --osd-data /var/lib/ceph/osd/ceph-2/ --osd-uuid 5fc7f603-a991-457e-876b-edcf7dd335a9 --setuser ceph --setgroup ceph
 stderr: 2025-07-03T12:13:33.922+0000 3ffb1a65900 -1 bluestore(/var/lib/ceph/osd/ceph-2//block) No valid bdev label found
 stderr: 2025-07-03T12:13:34.262+0000 3ffb1a65900 -1 bluestore(/var/lib/ceph/osd/ceph-2/) _read_fsid unparsable uuid
--> ceph-volume raw dmcrypt prepare successful for: /mnt/ocs-deviceset-localblock-0-data-0wgjtv
2025-07-03 12:13:39.262723 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log lvm list /mnt/ocs-deviceset-localblock-0-data-0wgjtv --format json
2025-07-03 12:13:40.269992 D | cephosd: {}
2025-07-03 12:13:40.270054 I | cephosd: 0 ceph-volume lvm osd devices configured on this node
2025-07-03 12:13:40.270070 D | exec: Running command: cryptsetup luksDump /mnt/ocs-deviceset-localblock-0-data-0wgjtv
2025-07-03 12:13:40.276490 E | cephosd: failed to determine if the encrypted block "/mnt/ocs-deviceset-localblock-0-data-0wgjtv" is from our cluster. failed to dump LUKS header for disk "/mnt/ocs-deviceset-localblock-0-data-0wgjtv". Device /mnt/ocs-deviceset-localblock-0-data-0wgjtv is not a valid LUKS device.: exit status 1
2025-07-03 12:13:40.276518 D | exec: Running command: stdbuf -oL ceph-volume --log-path /tmp/ceph-log raw list /mnt/ocs-deviceset-localblock-0-data-0wgjtv --format json
2025-07-03 12:13:40.618557 D | cephosd: {
    "5fc7f603-a991-457e-876b-edcf7dd335a9": {
        "ceph_fsid": "bc41b8f3-a8df-4b00-92fa-fe9663c60aea",
        "device": "/mnt/ocs-deviceset-localblock-0-data-0wgjtv",
        "osd_id": 2,
        "osd_uuid": "5fc7f603-a991-457e-876b-edcf7dd335a9",
        "type": "bluestore"
    }
}
2025-07-03 12:13:40.618669 D | exec: Running command: lsblk /mnt/ocs-deviceset-localblock-0-data-0wgjtv --bytes --nodeps --pairs --paths --output SIZE,ROTA,RO,TYPE,PKNAME,NAME,KNAME,MOUNTPOINT,FSTYPE
2025-07-03 12:13:40.623035 D | sys: lsblk output: "SIZE=\"536870912000\" ROTA=\"1\" RO=\"0\" TYPE=\"mpath\" PKNAME=\"\" NAME=\"/dev/mapper/mpathb\" KNAME=\"/dev/dm-3\" MOUNTPOINT=\"\" FSTYPE=\"\""
2025-07-03 12:13:40.623076 I | cephosd: setting device class "hdd" for device "/mnt/ocs-deviceset-localblock-0-data-0wgjtv"
2025-07-03 12:13:40.623086 I | cephosd: 1 ceph-volume raw osd devices configured on this node
2025-07-03 12:13:40.623140 I | cephosd: devices = [{ID:2 Cluster:ceph UUID:5fc7f603-a991-457e-876b-edcf7dd335a9 DevicePartUUID: DeviceClass:hdd BlockPath:/mnt/ocs-deviceset-localblock-0-data-0wgjtv MetadataPath: WalPath: SkipLVRelease:true Location:root=default host=worker-0-odf-ci-2-test-ocs LVBackedPV:false CVMode:raw Store:bluestore TopologyAffinity: Encrypted:false ExportService:false NodeName: PVCName:}]
[root@m4221001 ~]#