[kni@registry.kni-qe-18 ~]$ oc logs -n openshift-operators cluster-group-upgrades-controller-manager-7bd67f749-mf68f manager I0112 17:09:59.707095 1 request.go:668] Waited for 1.029695461s due to client-side throttling, not priority and fairness, request: GET:https://172.30.0.1:443/apis/operators.coreos.com/v2?timeout=32s 2023-01-12T17:10:03.337Z INFO controller-runtime.metrics metrics server is starting to listen {"addr": "127.0.0.1:8080"} 2023-01-12T17:10:03.388Z INFO setup starting manager 2023-01-12T17:10:03.388Z INFO controller-runtime.manager starting metrics server {"path": "/metrics"} I0112 17:10:03.388894 1 leaderelection.go:243] attempting to acquire leader lease openshift-operators/9a2365a3.openshift.io... I0112 17:10:22.073615 1 leaderelection.go:253] successfully acquired lease openshift-operators/9a2365a3.openshift.io 2023-01-12T17:10:22.073Z DEBUG controller-runtime.manager.events Normal {"object": {"kind":"ConfigMap","namespace":"openshift-operators","name":"9a2365a3.openshift.io","uid":"19b95a22-5c9a-412d-adba-ba02ba418014","apiVersion":"v1","resourceVersion":"107896204"}, "reason": "LeaderElection", "message": "cluster-group-upgrades-controller-manager-7bd67f749-mf68f_d097a548-b74b-42b5-a1ac-e56abc5b0a97 became leader"} 2023-01-12T17:10:22.073Z DEBUG controller-runtime.manager.events Normal {"object": {"kind":"Lease","namespace":"openshift-operators","name":"9a2365a3.openshift.io","uid":"594c16b6-354b-4483-b000-b47177937d15","apiVersion":"coordination.k8s.io/v1","resourceVersion":"107896206"}, "reason": "LeaderElection", "message": "cluster-group-upgrades-controller-manager-7bd67f749-mf68f_d097a548-b74b-42b5-a1ac-e56abc5b0a97 became leader"} 2023-01-12T17:10:22.073Z INFO controller-runtime.manager.controller.managedclusterForCGU Starting EventSource {"reconciler group": "cluster.open-cluster-management.io", "reconciler kind": "ManagedCluster", "source": "kind source: /, Kind="} 2023-01-12T17:10:22.073Z INFO controller-runtime.manager.controller.managedclusterForCGU Starting EventSource {"reconciler group": "cluster.open-cluster-management.io", "reconciler kind": "ManagedCluster", "source": "kind source: /, Kind="} 2023-01-12T17:10:22.073Z INFO controller-runtime.manager.controller.managedclusterForCGU Starting Controller {"reconciler group": "cluster.open-cluster-management.io", "reconciler kind": "ManagedCluster"} 2023-01-12T17:10:22.074Z INFO controller-runtime.manager.controller.clustergroupupgrade Starting EventSource {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "source": "kind source: /, Kind="} 2023-01-12T17:10:22.075Z INFO controller-runtime.manager.controller.clustergroupupgrade Starting EventSource {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "source": "kind source: policy.open-cluster-management.io/v1, Kind=Policy"} 2023-01-12T17:10:22.075Z INFO controller-runtime.manager.controller.clustergroupupgrade Starting Controller {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade"} 2023-01-12T17:10:22.177Z INFO controller-runtime.manager.controller.managedclusterForCGU Starting workers {"reconciler group": "cluster.open-cluster-management.io", "reconciler kind": "ManagedCluster", "worker count": 1} 2023-01-12T17:10:22.178Z INFO controller-runtime.manager.controller.clustergroupupgrade Starting workers {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "worker count": 1} 2023-01-12T17:10:22.178Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "local-cluster"} 2023-01-12T17:10:22.178Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ocp-edge87"} 2023-01-12T17:10:22.178Z INFO controllers.ManagedClusterForCGU cluster is ready {"Name": "local-cluster"} 2023-01-12T17:10:22.278Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ocp-edge87", "version": "98789528"} 2023-01-12T17:10:22.279Z INFO controllers.ManagedClusterForCGU WARN: No child policies found for cluster {"Name": "local-cluster"} 2023-01-12T17:10:22.280Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "ocp-edge87"} 2023-01-12T17:10:22.280Z INFO controllers.ManagedClusterForCGU ZTP for the cluster has completed. ztp-done label found. {"Name": "ocp-edge87"} 2023-01-12T17:10:22.280Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "ocp-edge88"} 2023-01-12T17:10:22.280Z INFO controllers.ManagedClusterForCGU ZTP for the cluster has completed. ztp-done label found. {"Name": "ocp-edge88"} 2023-01-12T17:10:22.295Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ocp-edge87", "requeueRightAway": false} 2023-01-12T17:10:22.295Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ocp-edge88"} 2023-01-12T17:10:22.395Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ocp-edge88", "version": "98772960"} 2023-01-12T17:10:22.403Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ocp-edge88", "requeueRightAway": false} 2023-01-12T17:30:41.213Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:30:41.314Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107939239"} 2023-01-12T17:30:41.332Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": true} 2023-01-12T17:30:41.337Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:30:41.438Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107939253"} 2023-01-12T17:30:41.447Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": true} 2023-01-12T17:30:41.457Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:30:41.558Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107939260"} 2023-01-12T17:30:41.558Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:30:41.563Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:30:41.563Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:30:41.663Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:30:41.663Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:30:41.663Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "nil"} 2023-01-12T17:30:41.663Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "PreparingToStart", "cluster": "ocp-edge87"} 2023-01-12T17:30:41.663Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "PreparingToStart", "condition": "entry", "cluster": "ocp-edge87", "nextState": "Starting"} 2023-01-12T17:30:41.694Z INFO controllers.ClusterGroupUpgrade [getBackupJobTemplateData] {"workload image": "registry.redhat.io/openshift4/topology-aware-lifecycle-manager-recovery-rhel8@sha256:f4faf984308d6d10c9431a97c4d40f5669b38038a14a0dfc291146d1f8f731ff"} 2023-01-12T17:30:41.694Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge87", "template": "backup-ns-delete"} 2023-01-12T17:30:41.701Z INFO controllers.ClusterGroupUpgrade [preparing] {"nextState returned": "Starting"} 2023-01-12T17:30:41.701Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"previousState": "PreparingToStart", "nextState": "Starting", "cluster": "ocp-edge87"} 2023-01-12T17:30:41.701Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "PreparingToStart", "cluster": "ocp-edge88"} 2023-01-12T17:30:41.701Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "PreparingToStart", "condition": "entry", "cluster": "ocp-edge88", "nextState": "Starting"} 2023-01-12T17:30:41.748Z INFO controllers.ClusterGroupUpgrade [getBackupJobTemplateData] {"workload image": "registry.redhat.io/openshift4/topology-aware-lifecycle-manager-recovery-rhel8@sha256:f4faf984308d6d10c9431a97c4d40f5669b38038a14a0dfc291146d1f8f731ff"} 2023-01-12T17:30:41.748Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge88", "template": "backup-ns-delete"} 2023-01-12T17:30:41.754Z INFO controllers.ClusterGroupUpgrade [preparing] {"nextState returned": "Starting"} 2023-01-12T17:30:41.754Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"previousState": "PreparingToStart", "nextState": "Starting", "cluster": "ocp-edge88"} 2023-01-12T17:30:41.771Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueAfter": 30} 2023-01-12T17:30:41.771Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:30:41.871Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107939273"} 2023-01-12T17:30:41.872Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:30:41.880Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:30:41.880Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:30:41.952Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:30:41.952Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:30:41.952Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 2 clusters,}"} 2023-01-12T17:30:41.952Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "Starting", "cluster": "ocp-edge87"} 2023-01-12T17:30:41.955Z INFO controllers.ClusterGroupUpgrade [starting] {"starting started condition: ": "DependenciesNotPresent"} 2023-01-12T17:30:41.955Z INFO controllers.ClusterGroupUpgrade [getBackupJobTemplateData] {"workload image": "registry.redhat.io/openshift4/topology-aware-lifecycle-manager-recovery-rhel8@sha256:f4faf984308d6d10c9431a97c4d40f5669b38038a14a0dfc291146d1f8f731ff"} 2023-01-12T17:30:41.955Z INFO controllers.ClusterGroupUpgrade [starting] {"conditions: ": "DependenciesNotPresent"} 2023-01-12T17:30:41.955Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge87", "template": "backup-ns-create"} 2023-01-12T17:30:41.967Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge87", "template": "backup-sa-create"} 2023-01-12T17:30:41.975Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge87", "template": "backup-crb-create"} 2023-01-12T17:30:41.983Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge87", "template": "view-backup-namespace"} 2023-01-12T17:30:41.989Z INFO controllers.ClusterGroupUpgrade [starting] {"nextState returned": "Starting"} 2023-01-12T17:30:41.989Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "Starting", "cluster": "ocp-edge88"} 2023-01-12T17:30:41.992Z INFO controllers.ClusterGroupUpgrade [starting] {"starting started condition: ": "DependenciesNotPresent"} 2023-01-12T17:30:41.992Z INFO controllers.ClusterGroupUpgrade [getBackupJobTemplateData] {"workload image": "registry.redhat.io/openshift4/topology-aware-lifecycle-manager-recovery-rhel8@sha256:f4faf984308d6d10c9431a97c4d40f5669b38038a14a0dfc291146d1f8f731ff"} 2023-01-12T17:30:41.992Z INFO controllers.ClusterGroupUpgrade [starting] {"conditions: ": "DependenciesNotPresent"} 2023-01-12T17:30:41.992Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge88", "template": "backup-ns-create"} 2023-01-12T17:30:42.002Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge88", "template": "backup-sa-create"} 2023-01-12T17:30:42.007Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge88", "template": "backup-crb-create"} 2023-01-12T17:30:42.016Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge88", "template": "view-backup-namespace"} 2023-01-12T17:30:42.025Z INFO controllers.ClusterGroupUpgrade [starting] {"nextState returned": "Starting"} 2023-01-12T17:30:42.036Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueAfter": 30} 2023-01-12T17:31:11.771Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:31:11.872Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107939273"} 2023-01-12T17:31:11.872Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:31:11.884Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:31:11.884Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:31:11.968Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:31:11.968Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:31:11.968Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 2 clusters,}"} 2023-01-12T17:31:11.968Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "Starting", "cluster": "ocp-edge87"} 2023-01-12T17:31:11.973Z INFO controllers.ClusterGroupUpgrade [checkViewProcessing] {"viewStatus": "Watching resources successfully"} 2023-01-12T17:31:11.977Z INFO controllers.ClusterGroupUpgrade [starting] {"starting started condition: ": "NoJobView"} 2023-01-12T17:31:11.977Z INFO controllers.ClusterGroupUpgrade [getBackupJobTemplateData] {"workload image": "registry.redhat.io/openshift4/topology-aware-lifecycle-manager-recovery-rhel8@sha256:f4faf984308d6d10c9431a97c4d40f5669b38038a14a0dfc291146d1f8f731ff"} 2023-01-12T17:31:11.977Z INFO controllers.ClusterGroupUpgrade [starting] {"conditions: ": "NoJobView"} 2023-01-12T17:31:11.977Z INFO controllers.ClusterGroupUpgrade [triggerbackup] {"currentState": "Starting", "condition": "NoJobFoundOnSpoke", "cluster": "ocp-edge87", "nextState": "Starting"} 2023-01-12T17:31:11.977Z INFO controllers.ClusterGroupUpgrade [getBackupJobTemplateData] {"workload image": "registry.redhat.io/openshift4/topology-aware-lifecycle-manager-recovery-rhel8@sha256:f4faf984308d6d10c9431a97c4d40f5669b38038a14a0dfc291146d1f8f731ff"} 2023-01-12T17:31:11.977Z INFO controllers.ClusterGroupUpgrade [deployBackupWorkload] {"getBackupJobTemplateData": "ocp-edge87", "spec": {"Cluster":"ocp-edge87","ResourceName":"","PlatformImage":"","Operators":{"Indexes":null,"PackagesAndChannels":null},"WorkloadImage":"registry.redhat.io/openshift4/topology-aware-lifecycle-manager-recovery-rhel8@sha256:f4faf984308d6d10c9431a97c4d40f5669b38038a14a0dfc291146d1f8f731ff","JobTimeout":480,"ViewUpdateIntervalSec":0}, "status": "success"} 2023-01-12T17:31:11.982Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge87", "template": "backup-job-create"} 2023-01-12T17:31:11.997Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge87", "template": "view-backup-job"} 2023-01-12T17:31:12.004Z INFO controllers.ClusterGroupUpgrade [starting] {"nextState returned": "Starting"} 2023-01-12T17:31:12.004Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "Starting", "cluster": "ocp-edge88"} 2023-01-12T17:31:12.008Z INFO controllers.ClusterGroupUpgrade [checkViewProcessing] {"viewStatus": "failed to get resource with err: namespaces \"openshift-talo-backup\" not found"} 2023-01-12T17:31:12.009Z INFO controllers.ClusterGroupUpgrade [starting] {"starting started condition: ": "DependenciesNotPresent"} 2023-01-12T17:31:12.009Z INFO controllers.ClusterGroupUpgrade [getBackupJobTemplateData] {"workload image": "registry.redhat.io/openshift4/topology-aware-lifecycle-manager-recovery-rhel8@sha256:f4faf984308d6d10c9431a97c4d40f5669b38038a14a0dfc291146d1f8f731ff"} 2023-01-12T17:31:12.009Z INFO controllers.ClusterGroupUpgrade [starting] {"conditions: ": "DependenciesNotPresent"} 2023-01-12T17:31:12.009Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge88", "template": "backup-ns-create"} 2023-01-12T17:31:12.024Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] Already exists {"cluster": "ocp-edge88", "template": "backup-ns-create"} 2023-01-12T17:31:12.025Z INFO controllers.ClusterGroupUpgrade [starting] {"nextState returned": "Starting"} 2023-01-12T17:31:12.034Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueAfter": 30} 2023-01-12T17:31:42.034Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:31:42.135Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107939273"} 2023-01-12T17:31:42.136Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:31:42.140Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:31:42.140Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:31:42.208Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:31:42.208Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:31:42.208Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 2 clusters,}"} 2023-01-12T17:31:42.209Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "Starting", "cluster": "ocp-edge87"} 2023-01-12T17:31:42.213Z INFO controllers.ClusterGroupUpgrade [checkViewProcessing] {"viewStatus": "Watching resources successfully"} 2023-01-12T17:31:42.217Z INFO controllers.ClusterGroupUpgrade [checkViewProcessing] {"viewStatus": "Watching resources successfully"} 2023-01-12T17:31:42.217Z INFO controllers.ClusterGroupUpgrade [getJobStatus] {"condition": "&JobCondition{Type:Failed,Status:True,LastProbeTime:2023-01-12 17:31:28 +0000 UTC,LastTransitionTime:2023-01-12 17:31:28 +0000 UTC,Reason:BackoffLimitExceeded,Message:Job has reached the specified backoff limit,}"} 2023-01-12T17:31:42.217Z INFO controllers.ClusterGroupUpgrade [getJobStatus] {"BackoffLimitExceeded": "Job failed"} 2023-01-12T17:31:42.217Z INFO controllers.ClusterGroupUpgrade [starting] {"starting started condition: ": "JobBackoffLimitExceeded"} 2023-01-12T17:31:42.217Z INFO controllers.ClusterGroupUpgrade [getBackupJobTemplateData] {"workload image": "registry.redhat.io/openshift4/topology-aware-lifecycle-manager-recovery-rhel8@sha256:f4faf984308d6d10c9431a97c4d40f5669b38038a14a0dfc291146d1f8f731ff"} 2023-01-12T17:31:42.217Z INFO controllers.ClusterGroupUpgrade [starting] {"conditions: ": "JobBackoffLimitExceeded"} 2023-01-12T17:31:42.217Z INFO controllers.ClusterGroupUpgrade [starting] {"nextState returned": "UnrecoverableError"} 2023-01-12T17:31:42.217Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"previousState": "Starting", "nextState": "UnrecoverableError", "cluster": "ocp-edge87"} 2023-01-12T17:31:42.217Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "Starting", "cluster": "ocp-edge88"} 2023-01-12T17:31:42.221Z INFO controllers.ClusterGroupUpgrade [checkViewProcessing] {"viewStatus": "failed to get resource with err: namespaces \"openshift-talo-backup\" not found"} 2023-01-12T17:31:42.221Z INFO controllers.ClusterGroupUpgrade [starting] {"starting started condition: ": "DependenciesNotPresent"} 2023-01-12T17:31:42.221Z INFO controllers.ClusterGroupUpgrade [getBackupJobTemplateData] {"workload image": "registry.redhat.io/openshift4/topology-aware-lifecycle-manager-recovery-rhel8@sha256:f4faf984308d6d10c9431a97c4d40f5669b38038a14a0dfc291146d1f8f731ff"} 2023-01-12T17:31:42.221Z INFO controllers.ClusterGroupUpgrade [starting] {"conditions: ": "DependenciesNotPresent"} 2023-01-12T17:31:42.221Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge88", "template": "backup-ns-create"} 2023-01-12T17:31:42.234Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge88", "template": "backup-sa-create"} 2023-01-12T17:31:42.240Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge88", "template": "backup-crb-create"} 2023-01-12T17:31:42.247Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge88", "template": "view-backup-namespace"} 2023-01-12T17:31:42.256Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] Already exists {"cluster": "ocp-edge88", "template": "view-backup-namespace"} 2023-01-12T17:31:42.256Z INFO controllers.ClusterGroupUpgrade [starting] {"nextState returned": "Starting"} 2023-01-12T17:31:42.267Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueAfter": 30} 2023-01-12T17:32:12.267Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:32:12.368Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107941457"} 2023-01-12T17:32:12.369Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:32:12.375Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:12.375Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:32:12.446Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:12.446Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:32:12.446Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 1 clusters,}"} 2023-01-12T17:32:12.446Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "UnrecoverableError", "cluster": "ocp-edge87"} 2023-01-12T17:32:12.446Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"cluster": "ocp-edge87", "final state": "UnrecoverableError"} 2023-01-12T17:32:12.446Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "Starting", "cluster": "ocp-edge88"} 2023-01-12T17:32:12.451Z INFO controllers.ClusterGroupUpgrade [checkViewProcessing] {"viewStatus": "Watching resources successfully"} 2023-01-12T17:32:12.454Z INFO controllers.ClusterGroupUpgrade [starting] {"starting started condition: ": "NoJobView"} 2023-01-12T17:32:12.454Z INFO controllers.ClusterGroupUpgrade [getBackupJobTemplateData] {"workload image": "registry.redhat.io/openshift4/topology-aware-lifecycle-manager-recovery-rhel8@sha256:f4faf984308d6d10c9431a97c4d40f5669b38038a14a0dfc291146d1f8f731ff"} 2023-01-12T17:32:12.454Z INFO controllers.ClusterGroupUpgrade [starting] {"conditions: ": "NoJobView"} 2023-01-12T17:32:12.454Z INFO controllers.ClusterGroupUpgrade [triggerbackup] {"currentState": "Starting", "condition": "NoJobFoundOnSpoke", "cluster": "ocp-edge88", "nextState": "Starting"} 2023-01-12T17:32:12.454Z INFO controllers.ClusterGroupUpgrade [getBackupJobTemplateData] {"workload image": "registry.redhat.io/openshift4/topology-aware-lifecycle-manager-recovery-rhel8@sha256:f4faf984308d6d10c9431a97c4d40f5669b38038a14a0dfc291146d1f8f731ff"} 2023-01-12T17:32:12.454Z INFO controllers.ClusterGroupUpgrade [deployBackupWorkload] {"getBackupJobTemplateData": "ocp-edge88", "spec": {"Cluster":"ocp-edge88","ResourceName":"","PlatformImage":"","Operators":{"Indexes":null,"PackagesAndChannels":null},"WorkloadImage":"registry.redhat.io/openshift4/topology-aware-lifecycle-manager-recovery-rhel8@sha256:f4faf984308d6d10c9431a97c4d40f5669b38038a14a0dfc291146d1f8f731ff","JobTimeout":480,"ViewUpdateIntervalSec":0}, "status": "success"} 2023-01-12T17:32:12.457Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge88", "template": "backup-job-create"} 2023-01-12T17:32:12.474Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge88", "template": "view-backup-job"} 2023-01-12T17:32:12.482Z INFO controllers.ClusterGroupUpgrade [starting] {"nextState returned": "Starting"} 2023-01-12T17:32:12.496Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueAfter": 30} 2023-01-12T17:32:42.497Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:32:42.598Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107942509"} 2023-01-12T17:32:42.598Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:32:42.606Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:42.606Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:32:42.672Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:42.672Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:32:42.672Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 1 clusters,}"} 2023-01-12T17:32:42.672Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "", "cluster": "ocp-edge87"} 2023-01-12T17:32:42.672Z ERROR controllers.ClusterGroupUpgrade reconcileBackup error {"error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:42.672Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} 2023-01-12T17:32:42.672Z ERROR controller-runtime.manager.controller.clustergroupupgrade Reconciler error {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "name": "generated-cgu-disk-full-multiple-spokes", "namespace": "talm-test", "error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:42.677Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:32:42.778Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107942509"} 2023-01-12T17:32:42.778Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:32:42.783Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:42.783Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:32:42.837Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:42.837Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:32:42.837Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 1 clusters,}"} 2023-01-12T17:32:42.837Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "", "cluster": "ocp-edge87"} 2023-01-12T17:32:42.837Z ERROR controllers.ClusterGroupUpgrade reconcileBackup error {"error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:42.837Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} 2023-01-12T17:32:42.838Z ERROR controller-runtime.manager.controller.clustergroupupgrade Reconciler error {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "name": "generated-cgu-disk-full-multiple-spokes", "namespace": "talm-test", "error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:42.848Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:32:42.948Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107942509"} 2023-01-12T17:32:42.949Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:32:42.955Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:42.955Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:32:43.049Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:43.049Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:32:43.049Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 1 clusters,}"} 2023-01-12T17:32:43.049Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "", "cluster": "ocp-edge87"} 2023-01-12T17:32:43.049Z ERROR controllers.ClusterGroupUpgrade reconcileBackup error {"error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:43.049Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} 2023-01-12T17:32:43.049Z ERROR controller-runtime.manager.controller.clustergroupupgrade Reconciler error {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "name": "generated-cgu-disk-full-multiple-spokes", "namespace": "talm-test", "error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:43.070Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:32:43.171Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107942509"} 2023-01-12T17:32:43.171Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:32:43.176Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:43.176Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:32:43.234Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:43.234Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:32:43.234Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 1 clusters,}"} 2023-01-12T17:32:43.234Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "", "cluster": "ocp-edge87"} 2023-01-12T17:32:43.234Z ERROR controllers.ClusterGroupUpgrade reconcileBackup error {"error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:43.234Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} 2023-01-12T17:32:43.234Z ERROR controller-runtime.manager.controller.clustergroupupgrade Reconciler error {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "name": "generated-cgu-disk-full-multiple-spokes", "namespace": "talm-test", "error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:43.275Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:32:43.375Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107942509"} 2023-01-12T17:32:43.376Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:32:43.382Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:43.382Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:32:43.441Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:43.441Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:32:43.441Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 1 clusters,}"} 2023-01-12T17:32:43.441Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "", "cluster": "ocp-edge87"} 2023-01-12T17:32:43.441Z ERROR controllers.ClusterGroupUpgrade reconcileBackup error {"error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:43.441Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} 2023-01-12T17:32:43.441Z ERROR controller-runtime.manager.controller.clustergroupupgrade Reconciler error {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "name": "generated-cgu-disk-full-multiple-spokes", "namespace": "talm-test", "error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:43.521Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:32:43.622Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107942509"} 2023-01-12T17:32:43.622Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:32:43.633Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:43.633Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:32:43.696Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:43.696Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:32:43.696Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 1 clusters,}"} 2023-01-12T17:32:43.696Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "", "cluster": "ocp-edge87"} 2023-01-12T17:32:43.696Z ERROR controllers.ClusterGroupUpgrade reconcileBackup error {"error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:43.696Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} 2023-01-12T17:32:43.697Z ERROR controller-runtime.manager.controller.clustergroupupgrade Reconciler error {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "name": "generated-cgu-disk-full-multiple-spokes", "namespace": "talm-test", "error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:43.857Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:32:43.958Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107942509"} 2023-01-12T17:32:43.959Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:32:43.967Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:43.967Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:32:44.028Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:44.028Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:32:44.028Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 1 clusters,}"} 2023-01-12T17:32:44.028Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "", "cluster": "ocp-edge87"} 2023-01-12T17:32:44.028Z ERROR controllers.ClusterGroupUpgrade reconcileBackup error {"error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:44.028Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} 2023-01-12T17:32:44.028Z ERROR controller-runtime.manager.controller.clustergroupupgrade Reconciler error {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "name": "generated-cgu-disk-full-multiple-spokes", "namespace": "talm-test", "error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:44.349Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:32:44.450Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107942509"} 2023-01-12T17:32:44.450Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:32:44.457Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:44.457Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:32:44.519Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:44.519Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:32:44.519Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 1 clusters,}"} 2023-01-12T17:32:44.519Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "", "cluster": "ocp-edge87"} 2023-01-12T17:32:44.519Z ERROR controllers.ClusterGroupUpgrade reconcileBackup error {"error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:44.519Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} 2023-01-12T17:32:44.519Z ERROR controller-runtime.manager.controller.clustergroupupgrade Reconciler error {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "name": "generated-cgu-disk-full-multiple-spokes", "namespace": "talm-test", "error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:45.160Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:32:45.260Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107942509"} 2023-01-12T17:32:45.261Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:32:45.269Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:45.269Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:32:45.344Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:45.344Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:32:45.344Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 1 clusters,}"} 2023-01-12T17:32:45.345Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "", "cluster": "ocp-edge87"} 2023-01-12T17:32:45.345Z ERROR controllers.ClusterGroupUpgrade reconcileBackup error {"error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:45.345Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} 2023-01-12T17:32:45.345Z ERROR controller-runtime.manager.controller.clustergroupupgrade Reconciler error {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "name": "generated-cgu-disk-full-multiple-spokes", "namespace": "talm-test", "error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:46.625Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:32:46.726Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107942509"} 2023-01-12T17:32:46.726Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:32:46.730Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:46.730Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:32:46.801Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:46.801Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:32:46.801Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 1 clusters,}"} 2023-01-12T17:32:46.801Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "", "cluster": "ocp-edge87"} 2023-01-12T17:32:46.801Z ERROR controllers.ClusterGroupUpgrade reconcileBackup error {"error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:46.801Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} 2023-01-12T17:32:46.802Z ERROR controller-runtime.manager.controller.clustergroupupgrade Reconciler error {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "name": "generated-cgu-disk-full-multiple-spokes", "namespace": "talm-test", "error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:49.362Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:32:49.463Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107942509"} 2023-01-12T17:32:49.463Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:32:49.471Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:49.471Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:32:49.554Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:49.555Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:32:49.555Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 1 clusters,}"} 2023-01-12T17:32:49.555Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "", "cluster": "ocp-edge87"} 2023-01-12T17:32:49.555Z ERROR controllers.ClusterGroupUpgrade reconcileBackup error {"error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:49.555Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} 2023-01-12T17:32:49.555Z ERROR controller-runtime.manager.controller.clustergroupupgrade Reconciler error {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "name": "generated-cgu-disk-full-multiple-spokes", "namespace": "talm-test", "error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:54.675Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:32:54.776Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107942509"} 2023-01-12T17:32:54.776Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:32:54.783Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:54.784Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:32:54.855Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:32:54.855Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:32:54.855Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 1 clusters,}"} 2023-01-12T17:32:54.855Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "", "cluster": "ocp-edge87"} 2023-01-12T17:32:54.855Z ERROR controllers.ClusterGroupUpgrade reconcileBackup error {"error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:32:54.855Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} 2023-01-12T17:32:54.856Z ERROR controller-runtime.manager.controller.clustergroupupgrade Reconciler error {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "name": "generated-cgu-disk-full-multiple-spokes", "namespace": "talm-test", "error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:33:05.096Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:33:05.197Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107942509"} 2023-01-12T17:33:05.197Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:33:05.201Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:33:05.201Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:33:05.252Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:33:05.252Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:33:05.252Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 1 clusters,}"} 2023-01-12T17:33:05.252Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "", "cluster": "ocp-edge87"} 2023-01-12T17:33:05.252Z ERROR controllers.ClusterGroupUpgrade reconcileBackup error {"error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:33:05.253Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} 2023-01-12T17:33:05.253Z ERROR controller-runtime.manager.controller.clustergroupupgrade Reconciler error {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "name": "generated-cgu-disk-full-multiple-spokes", "namespace": "talm-test", "error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:33:25.733Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:33:25.843Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107942509"} 2023-01-12T17:33:25.843Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:33:25.848Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:33:25.848Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:33:25.926Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:33:25.926Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:33:25.926Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 1 clusters,}"} 2023-01-12T17:33:25.926Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "", "cluster": "ocp-edge87"} 2023-01-12T17:33:25.926Z ERROR controllers.ClusterGroupUpgrade reconcileBackup error {"error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:33:25.926Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} 2023-01-12T17:33:25.926Z ERROR controller-runtime.manager.controller.clustergroupupgrade Reconciler error {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "name": "generated-cgu-disk-full-multiple-spokes", "namespace": "talm-test", "error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:34:06.887Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:34:06.988Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107942509"} 2023-01-12T17:34:06.988Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:34:06.994Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:34:06.994Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:34:07.069Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:34:07.069Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:34:07.069Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 1 clusters,}"} 2023-01-12T17:34:07.069Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "", "cluster": "ocp-edge87"} 2023-01-12T17:34:07.069Z ERROR controllers.ClusterGroupUpgrade reconcileBackup error {"error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:34:07.069Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} 2023-01-12T17:34:07.069Z ERROR controller-runtime.manager.controller.clustergroupupgrade Reconciler error {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "name": "generated-cgu-disk-full-multiple-spokes", "namespace": "talm-test", "error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:35:28.990Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:35:29.091Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107942509"} 2023-01-12T17:35:29.091Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:35:29.099Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:35:29.099Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:35:29.167Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:35:29.167Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:35:29.167Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 1 clusters,}"} 2023-01-12T17:35:29.167Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "", "cluster": "ocp-edge87"} 2023-01-12T17:35:29.167Z ERROR controllers.ClusterGroupUpgrade reconcileBackup error {"error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:35:29.167Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} 2023-01-12T17:35:29.167Z ERROR controller-runtime.manager.controller.clustergroupupgrade Reconciler error {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "name": "generated-cgu-disk-full-multiple-spokes", "namespace": "talm-test", "error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:38:13.009Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:38:13.109Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107942509"} 2023-01-12T17:38:13.110Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"common-config-policy":"ztp-common","common-subscriptions-policy":"ztp-common","generated-policy-disk-full-multiple-spokes":"talm-test","group-du-sno-config-policy":"ztp-group","ocp-edge87-config-policy":"ztp-site","ocp-edge88-config-policy":"ztp-site"}} 2023-01-12T17:38:13.121Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "generated-policy-disk-full-multiple-spokes", "clusters: ": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:38:13.121Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["ocp-edge87","ocp-edge88"]]} 2023-01-12T17:38:13.169Z INFO controllers.ClusterGroupUpgrade filterFailedPrecachingClusters: {"clustersList": ["ocp-edge87", "ocp-edge88"]} 2023-01-12T17:38:13.169Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2023-01-12T17:38:13.169Z INFO controllers.ClusterGroupUpgrade [reconcileBackup] {"FindStatusCondition": "&Condition{Type:BackupSuceeded,Status:False,ObservedGeneration:0,LastTransitionTime:2023-01-12 17:30:41 +0000 UTC,Reason:InProgress,Message:Backup in progress for 1 clusters,}"} 2023-01-12T17:38:13.169Z INFO controllers.ClusterGroupUpgrade [triggerBackup] {"currentState": "", "cluster": "ocp-edge87"} 2023-01-12T17:38:13.169Z ERROR controllers.ClusterGroupUpgrade reconcileBackup error {"error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:38:13.170Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} 2023-01-12T17:38:13.170Z ERROR controller-runtime.manager.controller.clustergroupupgrade Reconciler error {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "name": "generated-cgu-disk-full-multiple-spokes", "namespace": "talm-test", "error": "[triggerBackup] unknown state "} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2 /remote-source/app/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:214 2023-01-12T17:41:51.318Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:41:51.419Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "version": "107962866"} 2023-01-12T17:41:51.554Z INFO controllers.ClusterGroupUpgrade [createResourcesFromTemplates] {"cluster": "ocp-edge88", "template": "backup-ns-delete"} 2023-01-12T17:41:51.576Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} 2023-01-12T17:43:40.850Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes"} 2023-01-12T17:43:40.951Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "talm-test/generated-cgu-disk-full-multiple-spokes", "requeueRightAway": false} [kni@registry.kni-qe-18 ~]$