vsphere-problem-detector-metrics is not reporting for invalid datastore with both Intree/CSI provisioners for query CheckStorageClasses

4.19.0-0.nightly-2025-04-08-163320

Always

 1. Install cluster from flexy template with latest payload
profile: ipi-on-vsphere/versioned-installer-fips
2. Create sc with invalid datastore using intree/CSI provisioner, pvc, dep. 
3. Check vsphere problem detector metrics for query CheckStorageClasses

It is reporting 0 value      
 {
        "metric": {
          "__name__": "vsphere_cluster_check_errors",
          "check": "CheckStorageClasses",
          "container": "vsphere-problem-detector-operator",
          "endpoint": "vsphere-metrics",
          "instance": "10.130.1.49:8444",
          "job": "vsphere-problem-detector-metrics",
          "namespace": "openshift-cluster-storage-operator",
          "pod": "vsphere-problem-detector-operator-6885dddb54-2jr8g",
          "service": "vsphere-problem-detector-metrics"
        },
        "value": [
          1744201252.878,
          "0"
        ]
      },

It should report value 1
      {
        "metric": {
          "__name__": "vsphere_cluster_check_errors",
          "check": "CheckStorageClasses",
          "container": "vsphere-problem-detector-operator",
          "endpoint": "vsphere-metrics",
          "instance": "10.130.1.49:8444",
          "job": "vsphere-problem-detector-metrics",
          "namespace": "openshift-cluster-storage-operator",
          "pod": "vsphere-problem-detector-operator-6885dddb54-2jr8g",
          "service": "vsphere-problem-detector-metrics"
        },
        "value": [
          1744201252.878,
          "1"
        ]
      },

Metrics report is working correctly on 4.12, 4.13, 4.16 version with invalid datastore

if datastore not supportable from 4.19 then 4.13, 4.16, 4.19 pvc describe info is same on all versions
oc describe pvc -n testropatil
Warning ProvisioningFailed 15s (x5 over 30s) csi.vsphere.vmware.com_vmware-vsphere-csi-driver-controller-7d6d68ddbc-fs4vc_735a896a-2504-447d-b5ac-d24223cfcd43 failed to provision volume with StorageClass "mystorageclass-csidatastore": rpc error: code = InvalidArgument desc = parsing storage class parameters failed with error: invalid parameters :map[datastore:NONDatastore diskformat:thin] 

sc_csi.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: mystorageclass-csidatastore
provisioner: csi.vsphere.vmware.com
reclaimPolicy: Delete
volumeBindingMode: WaitForFirstConsumer
parameters:
  diskformat: thin
  datastore:  NONDatastore
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: mypvc-fs-csidatastore
  namespace: testropatil
spec:
  accessModes:
    - ReadWriteOnce
  volumeMode: Filesystem
  storageClassName: mystorageclass-csidatastore
  resources:
    requests:
      storage: 1Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: mydep-fs-csidatastore
  namespace: testropatil
spec:
  replicas: 1
  selector:
    matchLabels:
      app: hello-storage1
  template:
    metadata:
      labels:
        app: hello-storage1
    spec:
      containers:
      - name: hello-storage
        image: quay.io/openshifttest/hello-openshift@sha256:b1aabe8c8272f750ce757b6c4263a2712796297511e0c6df79144ee188933623
        ports:
        - containerPort: 80
        volumeMounts:
        - name: local
          mountPath: /mnt/storage
      volumes:
      - name: local
        persistentVolumeClaim:
          claimName: mypvc-fs-csidatastore


sc_intree.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: mystorageclass-intreedatastore
provisioner: kubernetes.io/vsphere-volume
reclaimPolicy: Delete
volumeBindingMode: Immediate
parameters:
  diskformat: thin
  datastore:  NONDatastore
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: mypvc-fs-intree
  namespace: testropatil
spec:
  accessModes:
    - ReadWriteOnce
  volumeMode: Filesystem
  storageClassName: mystorageclass-intreedatastore
  resources:
    requests:
      storage: 1Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: mydep-fs-intree
  namespace: testropatil
spec:
  replicas: 1
  selector:
    matchLabels:
      app: hello-storage1
  template:
    metadata:
      labels:
        app: hello-storage1
    spec:
      containers:
      - name: hello-storage
        image: quay.io/openshifttest/hello-openshift@sha256:b1aabe8c8272f750ce757b6c4263a2712796297511e0c6df79144ee188933623
        ports:
        - containerPort: 80
        volumeMounts:
        - name: local
          mountPath: /mnt/storage
      volumes:
      - name: local
        persistentVolumeClaim:
          claimName: mypvc-fs-intree

oc get pvc,pod -n testropatil 
NAME                                          STATUS    VOLUME   CAPACITY   ACCESS MODES   STORAGECLASS                  VOLUMEATTRIBUTESCLASS   AGE
persistentvolumeclaim/mypvc-fs-csidatastore   Pending                                      mystorageclass-csidatastore   <unset>                 27m

NAME                                         READY   STATUS    RESTARTS   AGE
pod/mydep-fs-csidatastore-597d6cd4c7-ffwcl   0/1     Pending   0          27m    

token=`oc create token prometheus-k8s -n openshift-monitoring`oc -n openshift-monitoring exec -c prometheus prometheus-k8s-0 -- curl -k -H "Authorization: Bearer $token" 'https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=vsphere_cluster_check_errors' | jq

      {
        "metric": {
          "__name__": "vsphere_cluster_check_errors",
          "check": "CheckStorageClasses",
          "container": "vsphere-problem-detector-operator",
          "endpoint": "vsphere-metrics",
          "instance": "10.128.0.40:8444",
          "job": "vsphere-problem-detector-metrics",
          "namespace": "openshift-cluster-storage-operator",
          "pod": "vsphere-problem-detector-operator-5567b9fb7c-jr5z4",
          "service": "vsphere-problem-detector-metrics"
        },
        "value": [
          1744200872.761,
          "1"
        ]
      }