RHACM4K-1705: Observability: Setup a Grafana develop instance [P1][Sev1][Observability][Integration] (grafana_dev/g0) /home/centos/workspace/qe-acm-automation-poc/obs_e2e_test_execution/tests/pkg/tests/observability_grafana_dev_test.go:21
I0405 00:11:44.924496 4914 observability_grafana_dev_test.go:26] the output of grafana-dev-test.sh: configmap/sample-dashboard created
secret/grafana-dev-config created
deployment.apps/grafana-dev created
service/grafana-dev created
serviceaccount/grafana-dev created
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
creationTimestamp: "2023-04-05T00:03:21Z"
labels:
observability.open-cluster-management.io/name: observability
name: open-cluster-management:grafana-crb-dev
ownerReferences:

• apiVersion: observability.open-cluster-management.io/v1beta2
  blockOwnerDeletion: true
  controller: true
  kind: MultiClusterObservability
  name: observability
  uid: 6be9c7dd-84d6-4e5b-9803-e8ed1d8dacaa
  resourceVersion: "766887"
  uid: 97c2e980-099f-4a90-9727-d001bd5f32e9
  roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: open-cluster-management:grafana
  subjects:
• kind: ServiceAccount
  name: grafana-dev
  namespace: open-cluster-management-observability
  clusterrolebinding.rbac.authorization.k8s.io/open-cluster-management:grafana-crb-dev created
  route.route.openshift.io/grafana-dev created
  persistentvolumeclaim/grafana-dev created
  oauthclient.oauth.openshift.io/grafana-proxy-client-dev created
  deployment.apps/grafana-dev patched
  service/grafana-dev patched
  route.route.openshift.io/grafana-dev patched
  oauthclient.oauth.openshift.io/grafana-proxy-client-dev patched
  clusterrolebinding.rbac.authorization.k8s.io/open-cluster-management:grafana-crb-dev patched
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Retrying in 10s for waiting for grafana-dev pod ready ...
  Failed waiting for grafana-dev pod ready in 300s
  I0405 00:11:44.925601 4914 utils.go:58] Create kubeclient for url https://api.acmqe-280-ec2.dev09.red-chesterfield.com:6443 using kubeconfig path /root/.kube/config
  I0405 00:11:44.925655 4914 utils.go:211] Kubeconfig path /root/.kube/config
  I0405 00:11:45.543073 4914 utils.go:73] Create kubeclient dynamic for url https://api.acmqe-280-ec2.dev09.red-chesterfield.com:6443 using kubeconfig path /root/.kube/config
  I0405 00:11:45.543105 4914 utils.go:211] Kubeconfig path /root/.kube/config
  I0405 00:11:45.557601 4914 mco_deploy.go:192] MCO spec: {
  "advanced": {
  "alertmanager": {
  "replicas": 2,
  "resources":
  Unknown macro: { "limits"}

  },
  "compact": {
  "resources":

  Unknown macro: { "limits"}

  ,
  "serviceAccountAnnotations": {
  "test.com/role-arn": "s3_role"
  }
  },
  "grafana": {
  "replicas": 3,
  "resources":

  Unknown macro: { "limits"}

  },
  "observatoriumAPI": {
  "replicas": 3,
  "resources":

  Unknown macro: { "limits"}

  },
  "query": {
  "replicas": 3,
  "resources":

  Unknown macro: { "limits"}

  ,
  "serviceAccountAnnotations": {
  "test.com/role-arn": "s3_role"
  }
  },
  "queryFrontend": {
  "replicas": 3,
  "resources":

  Unknown macro: { "limits"}

  },
  "queryFrontendMemcached": {
  "connectionLimit": 2048,
  "maxItemSize": "2m",
  "memoryLimitMb": 2048,
  "replicas": 3,
  "resources":

  Unknown macro: { "limits"}

  },
  "rbacQueryProxy": {
  "replicas": 3,
  "resources":

  Unknown macro: { "limits"}

  },
  "receive": {
  "replicas": 2,
  "resources":

  Unknown macro: { "limits"}

  ,
  "serviceAccountAnnotations": {
  "test.com/role-arn": "s3_role"
  }
  },
  "retentionConfig": {
  "blockDuration": "3h",
  "deleteDelay": "50h",
  "retentionInLocal": "5d",
  "retentionResolution1h": "31d",
  "retentionResolution5m": "15d",
  "retentionResolutionRaw": "6d"
  },
  "rule": {
  "replicas": 1,
  "resources":

  Unknown macro: { "limits"}

  ,
  "serviceAccountAnnotations": {
  "test.com/role-arn": "s3_role"
  }
  },
  "store": {
  "replicas": 3,
  "resources":

  Unknown macro: { "limits"}

  ,
  "serviceAccountAnnotations": {
  "test.com/role-arn": "s3_role"
  }
  },
  "storeMemcached": {
  "connectionLimit": 2048,
  "maxItemSize": "2m",
  "memoryLimitMb": 2048,
  "replicas": 2,
  "resources":

  Unknown macro: { "limits"}

  }
  },
  "enableDownsampling": true,
  "imagePullSecret": "multiclusterhub-operator-pull-secret",
  "nodeSelector": {
  "kubernetes.io/os": "linux"
  },
  "observabilityAddonSpec": {
  "enableMetrics": true,
  "interval": 30,
  "resources":

  Unknown macro: { "limits"}

  }
  },
  "storageConfig": {
  "alertmanagerStorageSize": "2Gi",
  "compactStorageSize": "2Gi",
  "metricObjectStorage":

  { "key": "thanos.yaml", "name": "thanos-object-storage" }

  ,
  "receiveStorageSize": "2Gi",
  "ruleStorageSize": "2Gi",
  "storageClass": "gp2",
  "storeStorageSize": "2Gi"
  }
  }
  I0405 00:11:45.557639 4914 mco_deploy.go:193] MCO status: {
  "conditions": [
  

  { "lastTransitionTime": "2023-04-05T00:03:25Z", "message": "Installation is in progress", "reason": "Installing", "status": "True", "type": "Installing" }

  ,
  {
  "lastTransitionTime": "2023-04-05T00:04:49Z",
  "message": "Observability components are deployed and running",
  "reason": "Ready",
  "status": "True",
  "type": "Ready"
  }
  ]
  }
  I0405 00:11:45.557659 4914 utils.go:58] Create kubeclient for url https://api.acmqe-280-ec2.dev09.red-chesterfield.com:6443 using kubeconfig path /root/.kube/config
  I0405 00:11:45.557667 4914 utils.go:211] Kubeconfig path /root/.kube/config
  I0405 00:11:45.593842 4914 mco_deploy.go:161] Get <32> pods in <open-cluster-management-observability> namespace
  I0405 00:11:45.593885 4914 mco_deploy.go:171] Pod <grafana-dev-67cd8b57db-vz4cs> is not <Ready> on <Pending> status due to v1.PodStatus{Phase:"Pending", Conditions:[]v1.PodCondition{v1.PodCondition{Type:"PodScheduled", Status:"False", LastProbeTime:time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), LastTransitionTime:time.Date(2023, time.April, 5, 0, 6, 34, 0, time.Local), Reason:"SchedulerError", Message:"running PreFilter plugin \"VolumeBinding\": %!w(<nil>)"}}, Message:"", Reason:"", NominatedNodeName:"", HostIP:"", PodIP:"", PodIPs:[]v1.PodIP(nil), StartTime:<nil>, InitContainerStatuses:[]v1.ContainerStatus(nil), ContainerStatuses:[]v1.ContainerStatus(nil), QOSClass:"Burstable", EphemeralContainerStatuses:[]v1.ContainerStatus(nil)}
  I0405 00:11:45.594034 4914 utils.go:58] Create kubeclient for url https://api.acmqe-280-ec2.dev09.red-chesterfield.com:6443 using kubeconfig path /root/.kube/config
  I0405 00:11:45.594040 4914 utils.go:211] Kubeconfig path /root/.kube/config

I0405 00:11:45.607172 4914 mco_deploy.go:232] Get <2> pods in <open-cluster-management-addon-observability> namespace from managedcluster*• Failure [326.166 seconds]/home/centos/workspace/qe-acm-automation-poc/obs_e2e_test_execution/tests/pkg/tests/observability_grafana_dev_test.go:17 *RHACM4K-1705: Observability: Setup a Grafana develop instance [P1][Sev1][Observability][Integration] (grafana_dev/g0) [It] /home/centos/workspace/qe-acm-automation-poc/obs_e2e_test_execution/tests/pkg/tests/observability_grafana_dev_test.go:21

Unexpected error:
<*exec.ExitError | 0xc00317e000>: {
ProcessState: {
pid: 5045,
status: 256,
rusage: {
Utime:

,
Stime: {Sec: 4, Usec: 726869},
Maxrss: 222368,
Ixrss: 0,
Idrss: 0,
Isrss: 0,
Minflt: 1033081,
Majflt: 3,
Nswap: 0,
Inblock: 2784,
Oublock: 477448,
Msgsnd: 0,
Msgrcv: 0,
Nsignals: 0,
Nvcsw: 30181,
Nivcsw: 1533,
},
},
Stderr: nil,
}
exit status 1
occurred

/home/centos/workspace/qe-acm-automation-poc/obs_e2e_test_execution/tests/pkg/tests/observability_grafana_dev_test.go:27