Creating this bug as a tracker for the Fusion HCI Issue in github
https://github.ibm.com/ProjectAbell/abell-tracking/issues/39567

Reporter - Dhanashree Patil

Description of problem:
===========================

storage client-status-reporter pod has been terminating for a few days, the force delete and patching the finalizer did not work either.

Could not fetch Logs

root@roadien10 ~]# oc logs storageclient-737342087af10580-status-reporter-28682138-mdm4w
unable to retrieve container logs for cri-o://83d987f7d96a86266c66e48984a8f8de71d2c57abf7eac75c0b1f62b4a24502c

containerStatuses:

• containerID: cri-o://83d987f7d96a86266c66e48984a8f8de71d2c57abf7eac75c0b1f62b4a24502c
  image: icr.io/cpopen/ocs-client-operator@sha256:ff5e5fee136854398e690ffe3c336bb8932e7f3d40460a54495e9d53e8918a14
  imageID: icr.io/cpopen/ocs-client-operator@sha256:d0f0b7180ec550ef41b572a27bae7b4a8d070bb8b402f44eff916825fe29a232
  lastState: {}
  name: heartbeat
  ready: false
  restartCount: 0
  started: false
  state:
  terminated:
  containerID: cri-o://83d987f7d96a86266c66e48984a8f8de71d2c57abf7eac75c0b1f62b4a24502c
  exitCode: 0
  finishedAt: "2024-07-14T03:38:55Z"
  reason: Completed
  startedAt: "2024-07-14T03:38:55Z"
  hostIP: 172.88.66.128
  hostIPs:
• ip: 172.88.66.128
  phase: Succeeded
  podIP: 10.131.0.152
  podIPs:
• ip: 10.131.0.152
  qosClass: Burstable
  startTime: "2024-07-14T03:38:54Z"
  > [root@roadien10 ~]# oc get co
  > NAME VERSION AVAILABLE PROGRESSING DEGRADED SINCE MESSAGE
  > authentication 4.16.0-rc.2 True False False 2d15h
  > baremetal 4.16.0-rc.2 True False False 37d
  > cloud-controller-manager 4.16.0-rc.2 True False False 37d
  > cloud-credential 4.16.0-rc.2 True False False 37d
  > cluster-autoscaler 4.16.0-rc.2 True False False 37d
  > config-operator 4.16.0-rc.2 True False False 37d
  > console 4.16.0-rc.2 True False False 2d15h
  > control-plane-machine-set 4.16.0-rc.2 True False False 37d
  > csi-snapshot-controller 4.16.0-rc.2 True False False 37d
  > dns 4.16.0-rc.2 True False False 37d
  > etcd 4.16.0-rc.2 True False False 37d
  > image-registry 4.16.0-rc.2 True False False 37d
  > ingress 4.16.0-rc.2 True False False 2d21h
  > insights 4.16.0-rc.2 True False False 37d
  > kube-apiserver 4.16.0-rc.2 True False False 37d
  > kube-controller-manager 4.16.0-rc.2 True False False 37d
  > kube-scheduler 4.16.0-rc.2 True False False 37d
  > kube-storage-version-migrator 4.16.0-rc.2 True False False 24d
  > machine-api 4.16.0-rc.2 True False False 37d
  > machine-approver 4.16.0-rc.2 True False False 37d
  > machine-config 4.16.0-rc.2 True False False 37d
  > marketplace 4.16.0-rc.2 True False False 37d
  > monitoring 4.16.0-rc.2 True False False 23d
  > network 4.16.0-rc.2 True False False 37d
  > node-tuning 4.16.0-rc.2 True False False 2d21h
  > openshift-apiserver 4.16.0-rc.2 True False False 2d21h
  > openshift-controller-manager 4.16.0-rc.2 True False False 37d
  > openshift-samples 4.16.0-rc.2 True False False 37d
  > operator-lifecycle-manager 4.16.0-rc.2 True False False 37d
  > operator-lifecycle-manager-catalog 4.16.0-rc.2 True False False 37d
  > operator-lifecycle-manager-packageserver 4.16.0-rc.2 True False False 24d
  > service-ca 4.16.0-rc.2 True False False 37d
  > storage 4.16.0-rc.2 True False False 37d

ISF- 2.8.1-21415544
FDF- v4.16.0-142

[root@roadien10 ~]# oc get job,cronjob

NAME COMPLETIONS DURATION AGE
job.batch/rook-ceph-osd-prepare-3596336d9c2da345ae5e85803d892e5a 1/1 9s 24d
job.batch/rook-ceph-osd-prepare-390c0ee858e533998f3266dbd4e4849b 1/1 9s 25d
job.batch/rook-ceph-osd-prepare-434ce1d95f485de086db6e4ea5673ad2 1/1 9s 24d
job.batch/rook-ceph-osd-prepare-519257dbd0376c8e916840be2400e0af 1/1 9s 25d
job.batch/rook-ceph-osd-prepare-716f6300df51f06e4f7518a689f88a9d 1/1 9s 24d
job.batch/rook-ceph-osd-prepare-99ab06da3e4b22dc91426fbc6d8880a3 1/1 9s 25d
job.batch/rook-ceph-osd-prepare-a12f8e6e060309df340bd4988c95501d 1/1 9s 25d
job.batch/rook-ceph-osd-prepare-b8928cc1245ed3449bbe8ec6eb29508d 1/1 9s 24d
job.batch/rook-ceph-osd-prepare-cd59e9b57e21e8d8bda0a4e68459f4f8 1/1 19s 25d
job.batch/storageclient-737342087af10580-status-reporter-28682138 0/1 8d 8d

NAME SCHEDULE SUSPEND ACTIVE LAST SCHEDULE AGE
cronjob.batch/storageclient-737342087af10580-status-reporter * * * * * False 1 8d 25d