I1123 12:06:20.594771 1 request.go:668] Waited for 1.01260127s due to client-side throttling, not priority and fairness, request: GET:https://172.30.0.1:443/apis/agent-install.openshift.io/v1beta1?timeout=32s I1123 12:06:30.642432 1 request.go:668] Waited for 5.747463753s due to client-side throttling, not priority and fairness, request: GET:https://172.30.0.1:443/apis/agent-install.openshift.io/v1beta1?timeout=32s 2022-11-23T12:06:31.548Z INFO controller-runtime.metrics metrics server is starting to listen {"addr": "127.0.0.1:8080"} 2022-11-23T12:06:31.561Z INFO setup starting manager I1123 12:06:31.565996 1 leaderelection.go:243] attempting to acquire leader lease openshift-talm/9a2365a3.openshift.io... 2022-11-23T12:06:31.566Z INFO controller-runtime.manager starting metrics server {"path": "/metrics"} I1123 12:06:49.867237 1 leaderelection.go:253] successfully acquired lease openshift-talm/9a2365a3.openshift.io 2022-11-23T12:06:49.867Z DEBUG controller-runtime.manager.events Normal {"object": {"kind":"ConfigMap","namespace":"openshift-talm","name":"9a2365a3.openshift.io","uid":"12d1336f-5c6f-4aab-a252-683dac5565d2","apiVersion":"v1","resourceVersion":"3080594"}, "reason": "LeaderElection", "message": "cluster-group-upgrades-controller-manager-7b64b5bbbd-xl57w_df88c4d4-3e6b-4a6f-8e4d-3ec084b64eac became leader"} 2022-11-23T12:06:49.867Z DEBUG controller-runtime.manager.events Normal {"object": {"kind":"Lease","namespace":"openshift-talm","name":"9a2365a3.openshift.io","uid":"8789172f-9da0-451b-8e6f-a8bff150a555","apiVersion":"coordination.k8s.io/v1","resourceVersion":"3080596"}, "reason": "LeaderElection", "message": "cluster-group-upgrades-controller-manager-7b64b5bbbd-xl57w_df88c4d4-3e6b-4a6f-8e4d-3ec084b64eac became leader"} 2022-11-23T12:06:49.868Z INFO controller-runtime.manager.controller.managedclusterForCGU Starting EventSource {"reconciler group": "cluster.open-cluster-management.io", "reconciler kind": "ManagedCluster", "source": "kind source: /, Kind="} 2022-11-23T12:06:49.868Z INFO controller-runtime.manager.controller.clustergroupupgrade Starting EventSource {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "source": "kind source: /, Kind="} 2022-11-23T12:06:49.868Z INFO controller-runtime.manager.controller.managedclusterForCGU Starting EventSource {"reconciler group": "cluster.open-cluster-management.io", "reconciler kind": "ManagedCluster", "source": "kind source: /, Kind="} 2022-11-23T12:06:49.868Z INFO controller-runtime.manager.controller.managedclusterForCGU Starting Controller {"reconciler group": "cluster.open-cluster-management.io", "reconciler kind": "ManagedCluster"} 2022-11-23T12:06:49.868Z INFO controller-runtime.manager.controller.clustergroupupgrade Starting EventSource {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "source": "kind source: policy.open-cluster-management.io/v1, Kind=Policy"} 2022-11-23T12:06:49.868Z INFO controller-runtime.manager.controller.clustergroupupgrade Starting Controller {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade"} 2022-11-23T12:06:49.969Z INFO controller-runtime.manager.controller.managedclusterForCGU Starting workers {"reconciler group": "cluster.open-cluster-management.io", "reconciler kind": "ManagedCluster", "worker count": 1} 2022-11-23T12:06:49.969Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site1"} 2022-11-23T12:06:49.969Z INFO controllers.ManagedClusterForCGU ZTP for the cluster has completed. ztp-done label found. {"Name": "cloudransno-site1"} 2022-11-23T12:06:49.969Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site2"} 2022-11-23T12:06:49.969Z INFO controllers.ManagedClusterForCGU ZTP for the cluster has completed. ztp-done label found. {"Name": "cloudransno-site2"} 2022-11-23T12:06:49.969Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site3"} 2022-11-23T12:06:49.969Z INFO controllers.ManagedClusterForCGU ZTP for the cluster has completed. ztp-done label found. {"Name": "cloudransno-site3"} 2022-11-23T12:06:49.969Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site4"} 2022-11-23T12:06:49.969Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:06:49.969Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site5"} 2022-11-23T12:06:49.969Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:06:49.969Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site6"} 2022-11-23T12:06:49.969Z INFO controllers.ManagedClusterForCGU clusterGroupUpgrade found {"Name": "cloudransno-site6", "Namespace": "ztp-install"} 2022-11-23T12:06:49.969Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "local-cluster"} 2022-11-23T12:06:49.969Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:06:49.969Z INFO controller-runtime.manager.controller.clustergroupupgrade Starting workers {"reconciler group": "ran.openshift.io", "reconciler kind": "ClusterGroupUpgrade", "worker count": 1} 2022-11-23T12:06:49.969Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/cloudransno-site1"} 2022-11-23T12:06:50.072Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/cloudransno-site1", "version": "1576929"} 2022-11-23T12:06:50.114Z INFO controllers.ClusterGroupUpgrade Upgrade is completed 2022-11-23T12:06:50.124Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/cloudransno-site1", "requeueRightAway": false} 2022-11-23T12:06:50.124Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/cloudransno-site2"} 2022-11-23T12:06:50.225Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/cloudransno-site2", "version": "1530818"} 2022-11-23T12:06:50.225Z INFO controllers.ClusterGroupUpgrade Upgrade is completed 2022-11-23T12:06:50.233Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/cloudransno-site2", "requeueRightAway": false} 2022-11-23T12:06:50.233Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/cloudransno-site3"} 2022-11-23T12:06:50.333Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/cloudransno-site3", "version": "1619137"} 2022-11-23T12:06:50.333Z INFO controllers.ClusterGroupUpgrade Upgrade is completed 2022-11-23T12:06:50.341Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/cloudransno-site3", "requeueRightAway": false} 2022-11-23T12:06:50.341Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/cloudransno-site6"} 2022-11-23T12:06:50.442Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/cloudransno-site6", "version": "2344680"} 2022-11-23T12:06:50.442Z INFO controllers.ClusterGroupUpgrade CGU has timed out 2022-11-23T12:06:50.442Z DEBUG controller-runtime.manager.events Warning {"object": {"kind":"ClusterGroupUpgrade","namespace":"ztp-install","name":"cloudransno-site6","uid":"86ce198f-f00a-4791-b2df-d5e9b73bc8bc","apiVersion":"ran.openshift.io/v1alpha1","resourceVersion":"2344680"}, "reason": "UpgradeTimedOut", "message": "The ClusterGroupUpgrade CR policies are taking too long to complete"} 2022-11-23T12:06:50.753Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/cloudransno-site6", "requeueRightAway": false} 2022-11-23T12:06:50.754Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T12:06:50.854Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3055383"} 2022-11-23T12:06:50.854Z INFO controllers.ClusterGroupUpgrade Upgrade is completed 2022-11-23T12:06:50.860Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": false} 2022-11-23T12:11:49.970Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site4"} 2022-11-23T12:11:49.970Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:11:49.970Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site5"} 2022-11-23T12:11:49.970Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:11:49.970Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "local-cluster"} 2022-11-23T12:11:49.970Z INFO controllers.ManagedClusterForCGU cluster is ready {"Name": "local-cluster"} 2022-11-23T12:11:49.970Z INFO controllers.ManagedClusterForCGU WARN: No child policies found for cluster {"Name": "local-cluster"} 2022-11-23T12:16:49.971Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site4"} 2022-11-23T12:16:49.971Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:16:49.971Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site5"} 2022-11-23T12:16:49.971Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:21:49.972Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site4"} 2022-11-23T12:21:49.972Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:21:49.972Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site5"} 2022-11-23T12:21:49.972Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:26:49.973Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site4"} 2022-11-23T12:26:49.973Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:26:49.973Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site5"} 2022-11-23T12:26:49.973Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:31:49.974Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site4"} 2022-11-23T12:31:49.974Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:31:49.974Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site5"} 2022-11-23T12:31:49.974Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:36:49.975Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site4"} 2022-11-23T12:36:49.975Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:36:49.975Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site5"} 2022-11-23T12:36:49.975Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:41:49.976Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site4"} 2022-11-23T12:41:49.976Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:41:49.976Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site5"} 2022-11-23T12:41:49.976Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:46:49.977Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site4"} 2022-11-23T12:46:49.977Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:46:49.977Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site5"} 2022-11-23T12:46:49.977Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:51:49.978Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site4"} 2022-11-23T12:51:49.978Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:51:49.978Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site5"} 2022-11-23T12:51:49.978Z INFO controllers.ManagedClusterForCGU cluster is not ready {"RequeueAfter:": "5m0s"} 2022-11-23T12:56:49.979Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site4"} 2022-11-23T12:56:49.979Z INFO controllers.ManagedClusterForCGU cluster is ready {"Name": "cloudransno-site4"} 2022-11-23T12:56:49.979Z INFO controllers.ManagedClusterForCGU WARN: No child policies found for cluster {"Name": "cloudransno-site4"} 2022-11-23T12:56:49.979Z INFO controllers.ManagedClusterForCGU Reconciling managedCluster to create clusterGroupUpgrade {"Request.Name": "cloudransno-site5"} 2022-11-23T12:56:49.979Z INFO controllers.ManagedClusterForCGU cluster is ready {"Name": "cloudransno-site5"} 2022-11-23T12:56:49.979Z INFO controllers.ManagedClusterForCGU WARN: No child policies found for cluster {"Name": "cloudransno-site5"} 2022-11-23T13:03:48.820Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:03:48.920Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3289261"} 2022-11-23T13:03:48.927Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": false} 2022-11-23T13:03:49.426Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:03:49.527Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3289296"} 2022-11-23T13:03:49.533Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": true} 2022-11-23T13:03:49.533Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:03:49.633Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3289299"} 2022-11-23T13:03:49.640Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": true} 2022-11-23T13:03:49.640Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:03:49.740Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3289306"} 2022-11-23T13:03:49.745Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": true} 2022-11-23T13:03:49.745Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:03:49.846Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3289310"} 2022-11-23T13:03:49.847Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"cloudransno-site1-perfprofile-policy":"ztp-policies","cloudransno-site1-ptp-config-policy":"ztp-policies","cloudransno-site1-sriov-dpdk-policy":"ztp-policies","cloudransno-site1-sriov-nw-dpdk-policy":"ztp-policies","cloudransno-site1-tuned-perf-patch-policy":"ztp-policies","cloudransno-site2-perfprofile-policy":"ztp-policies","cloudransno-site2-ptp-config-policy":"ztp-policies","cloudransno-site2-sriov-nw-ping-policy":"ztp-policies","cloudransno-site2-sriov-ping-policy":"ztp-policies","cloudransno-site2-tuned-perf-patch-policy":"ztp-policies","cloudransno-site3-console-policy":"ztp-policies","cloudransno-site3-perfprofile-policy":"ztp-policies","cloudransno-site3-ptp-config-policy":"ztp-policies","cloudransno-site3-sriov-nw-ping-policy":"ztp-policies","cloudransno-site3-sriov-ping-policy":"ztp-policies","cloudransno-site3-tuned-perf-patch-policy":"ztp-policies","cloudransno-site4-perf-policy":"ztp-policies","cloudransno-site4-perfprofile-policy":"ztp-policies","cloudransno-site4-sriov-dpdk-policy":"ztp-policies","cloudransno-site4-sriov-nw-dpdk-policy":"ztp-policies","cloudransno-site5-perf-policy":"ztp-policies","cloudransno-site5-perfprofile-policy":"ztp-policies","cloudransno-site5-sriov-nw-ping-policy":"ztp-policies","cloudransno-site5-sriov-ping-policy":"ztp-policies","cloudransno-site6-fec-policy":"ztp-policies","cloudransno-site6-perf-policy":"ztp-policies","cloudransno-site6-perfprofile-policy":"ztp-policies","cloudransno-site6-sriov-nw-ping-policy":"ztp-policies","cloudransno-site6-sriov-ping-policy":"ztp-policies","common-kdump-policy":"ztp-common","common-mon-offload-policy":"ztp-common","common-pao-sub-policy":"ztp-common","common-ptp-sub-policy":"ztp-common","common-sriov-sub-policy":"ztp-common","common-storage-sub-policy":"ztp-common","common-subscriptions-policy":"ztp-common","group-du-sno-mc-sctp-policy":"ztp-group-du-sno"}} 2022-11-23T13:03:49.850Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "cloudransno-site3-sriov-nw-ping-policy", "clusters: ": ["cloudransno-site5", "cloudransno-site6", "cloudransno-site1", "cloudransno-site2", "cloudransno-site3", "cloudransno-site4"]} 2022-11-23T13:03:49.853Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "cloudransno-site3-sriov-ping-policy", "clusters: ": ["cloudransno-site6", "cloudransno-site1", "cloudransno-site2", "cloudransno-site3", "cloudransno-site4", "cloudransno-site5"]} 2022-11-23T13:03:49.853Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["cloudransno-site3"]]} 2022-11-23T13:03:49.938Z INFO controllers.ClusterGroupUpgrade [getPolicyContent] Policy spec.policy-templates.objectDefinition.spec.object-templates.kind is not of Subscription kind {"policyName": "cloudransno-site3-sriov-nw-ping-policy"} 2022-11-23T13:03:49.938Z INFO controllers.ClusterGroupUpgrade [getPolicyContent] Policy spec.policy-templates.objectDefinition.spec.object-templates.kind is not of Subscription kind {"policyName": "cloudransno-site3-sriov-ping-policy"} 2022-11-23T13:03:49.938Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2022-11-23T13:03:49.944Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": true} 2022-11-23T13:03:49.944Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:03:50.045Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3289336"} 2022-11-23T13:03:50.045Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 0} 2022-11-23T13:03:50.045Z INFO controllers.ClusterGroupUpgrade [initializeRemediationPolicyForBatch] {"CurrentBatchRemediationProgress": {"cloudransno-site3":{"state":"NotStarted"}}} 2022-11-23T13:03:50.048Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site3":{"state":"InProgress","policyIndex":0}}, "isBatchComplete": false} 2022-11-23T13:03:50.065Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": true} 2022-11-23T13:03:50.065Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:03:50.166Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3289347"} 2022-11-23T13:03:50.167Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-23T13:03:50.171Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site3":{"state":"InProgress","policyIndex":0}}, "isBatchComplete": false} 2022-11-23T13:03:50.184Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueAfter": 299.832788777} 2022-11-23T13:03:50.184Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:03:50.284Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3289347"} 2022-11-23T13:03:50.284Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-23T13:03:50.288Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site3":{"state":"InProgress","policyIndex":0}}, "isBatchComplete": false} 2022-11-23T13:03:50.304Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueAfter": 299.715260807} 2022-11-23T13:03:50.304Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:03:50.404Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3289347"} 2022-11-23T13:03:50.404Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-23T13:03:50.409Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site3":{"state":"InProgress","policyIndex":0}}, "isBatchComplete": false} 2022-11-23T13:03:50.422Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueAfter": 299.59521102} 2022-11-23T13:04:00.976Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:04:01.076Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3289347"} 2022-11-23T13:04:01.077Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-23T13:04:01.079Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site3":{"state":"InProgress","policyIndex":0}}, "isBatchComplete": false} 2022-11-23T13:04:01.094Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueAfter": 288.922932659} 2022-11-23T13:08:50.017Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:08:50.117Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3289347"} 2022-11-23T13:08:50.118Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-23T13:08:50.126Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site3":{"state":"InProgress","policyIndex":1}}, "isBatchComplete": false} 2022-11-23T13:08:50.140Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueAfter": 300} 2022-11-23T13:08:50.162Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:08:50.263Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3307884"} 2022-11-23T13:08:50.263Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-23T13:08:50.266Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site3":{"state":"InProgress","policyIndex":1}}, "isBatchComplete": false} 2022-11-23T13:08:50.281Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueAfter": 300} 2022-11-23T13:08:58.283Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:08:58.383Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3307884"} 2022-11-23T13:08:58.383Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-23T13:08:58.386Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site3":{"state":"InProgress","policyIndex":1}}, "isBatchComplete": false} 2022-11-23T13:08:58.402Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueAfter": 300} 2022-11-23T13:13:50.140Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:13:50.240Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3307884"} 2022-11-23T13:13:50.241Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-23T13:13:50.245Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site3":{"state":"Completed"}}, "isBatchComplete": true} 2022-11-23T13:13:50.252Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": true} 2022-11-23T13:13:50.258Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:13:50.358Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3326027"} 2022-11-23T13:13:50.358Z INFO controllers.ClusterGroupUpgrade Upgrade is completed 2022-11-23T13:13:50.405Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": false} 2022-11-23T13:13:50.405Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:13:50.505Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3326043"} 2022-11-23T13:13:50.505Z INFO controllers.ClusterGroupUpgrade Upgrade is completed 2022-11-23T13:13:50.511Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": false} 2022-11-23T13:18:32.109Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:18:32.209Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3343074"} 2022-11-23T13:18:32.217Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": false} 2022-11-23T13:18:32.488Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:18:32.589Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3343097"} 2022-11-23T13:18:32.594Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": true} 2022-11-23T13:18:32.594Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:18:32.695Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3343102"} 2022-11-23T13:18:32.700Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": true} 2022-11-23T13:18:32.700Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:18:32.801Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3343109"} 2022-11-23T13:18:32.806Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": true} 2022-11-23T13:18:32.807Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:18:32.908Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3343112"} 2022-11-23T13:18:32.908Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"cloudransno-site1-perfprofile-policy":"ztp-policies","cloudransno-site1-ptp-config-policy":"ztp-policies","cloudransno-site1-sriov-dpdk-policy":"ztp-policies","cloudransno-site1-sriov-nw-dpdk-policy":"ztp-policies","cloudransno-site1-tuned-perf-patch-policy":"ztp-policies","cloudransno-site2-perfprofile-policy":"ztp-policies","cloudransno-site2-ptp-config-policy":"ztp-policies","cloudransno-site2-sriov-nw-ping-policy":"ztp-policies","cloudransno-site2-sriov-ping-policy":"ztp-policies","cloudransno-site2-tuned-perf-patch-policy":"ztp-policies","cloudransno-site3-console-policy":"ztp-policies","cloudransno-site3-perfprofile-policy":"ztp-policies","cloudransno-site3-ptp-config-policy":"ztp-policies","cloudransno-site3-sriov-nw-ping-policy":"ztp-policies","cloudransno-site3-sriov-ping-policy":"ztp-policies","cloudransno-site3-tuned-perf-patch-policy":"ztp-policies","cloudransno-site4-perf-policy":"ztp-policies","cloudransno-site4-perfprofile-policy":"ztp-policies","cloudransno-site4-sriov-dpdk-policy":"ztp-policies","cloudransno-site4-sriov-nw-dpdk-policy":"ztp-policies","cloudransno-site5-perf-policy":"ztp-policies","cloudransno-site5-perfprofile-policy":"ztp-policies","cloudransno-site5-sriov-nw-ping-policy":"ztp-policies","cloudransno-site5-sriov-ping-policy":"ztp-policies","cloudransno-site6-fec-policy":"ztp-policies","cloudransno-site6-perf-policy":"ztp-policies","cloudransno-site6-perfprofile-policy":"ztp-policies","cloudransno-site6-sriov-nw-ping-policy":"ztp-policies","cloudransno-site6-sriov-ping-policy":"ztp-policies","common-kdump-policy":"ztp-common","common-mon-offload-policy":"ztp-common","common-pao-sub-policy":"ztp-common","common-ptp-sub-policy":"ztp-common","common-sriov-sub-policy":"ztp-common","common-storage-sub-policy":"ztp-common","common-subscriptions-policy":"ztp-common","group-du-sno-mc-sctp-policy":"ztp-group-du-sno"}} 2022-11-23T13:18:32.912Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "cloudransno-site3-sriov-nw-ping-policy", "clusters: ": ["cloudransno-site5", "cloudransno-site6", "cloudransno-site1", "cloudransno-site2", "cloudransno-site3", "cloudransno-site4"]} 2022-11-23T13:18:32.915Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "cloudransno-site3-sriov-ping-policy", "clusters: ": ["cloudransno-site4", "cloudransno-site5", "cloudransno-site6", "cloudransno-site1", "cloudransno-site2"]} 2022-11-23T13:18:32.915Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["cloudransno-site3"]]} 2022-11-23T13:18:33.002Z INFO controllers.ClusterGroupUpgrade [getPolicyContent] Policy spec.policy-templates.objectDefinition.spec.object-templates.kind is not of Subscription kind {"policyName": "cloudransno-site3-sriov-nw-ping-policy"} 2022-11-23T13:18:33.002Z INFO controllers.ClusterGroupUpgrade [getPolicyContent] Policy spec.policy-templates.objectDefinition.spec.object-templates.kind is not of Subscription kind {"policyName": "cloudransno-site3-sriov-ping-policy"} 2022-11-23T13:18:33.002Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2022-11-23T13:18:33.009Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": true} 2022-11-23T13:18:33.009Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:18:33.109Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3343129"} 2022-11-23T13:18:33.110Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 0} 2022-11-23T13:18:33.110Z INFO controllers.ClusterGroupUpgrade [initializeRemediationPolicyForBatch] {"CurrentBatchRemediationProgress": {"cloudransno-site3":{"state":"NotStarted"}}} 2022-11-23T13:18:33.113Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site3":{"state":"InProgress","policyIndex":0}}, "isBatchComplete": false} 2022-11-23T13:18:33.128Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": true} 2022-11-23T13:18:33.128Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:18:33.229Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3343136"} 2022-11-23T13:18:33.229Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-23T13:18:33.234Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site3":{"state":"InProgress","policyIndex":0}}, "isBatchComplete": false} 2022-11-23T13:18:33.248Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueAfter": 299.770390708} 2022-11-23T13:18:33.248Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:18:33.348Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3343136"} 2022-11-23T13:18:33.349Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-23T13:18:33.352Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site3":{"state":"InProgress","policyIndex":0}}, "isBatchComplete": false} 2022-11-23T13:18:33.365Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueAfter": 299.650801004} 2022-11-23T13:18:38.888Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:18:38.988Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3343136"} 2022-11-23T13:18:38.988Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-23T13:18:38.992Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site3":{"state":"InProgress","policyIndex":0}}, "isBatchComplete": false} 2022-11-23T13:18:39.007Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueAfter": 294.011161169} 2022-11-23T13:23:33.017Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:23:33.117Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3343136"} 2022-11-23T13:23:33.117Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-23T13:23:33.123Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site3":{"state":"InProgress","policyIndex":1}}, "isBatchComplete": false} 2022-11-23T13:23:33.138Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueAfter": 300} 2022-11-23T13:23:33.157Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:23:33.258Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3361354"} 2022-11-23T13:23:33.258Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-23T13:23:33.264Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site3":{"state":"InProgress","policyIndex":1}}, "isBatchComplete": false} 2022-11-23T13:23:33.281Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueAfter": 300} 2022-11-23T13:23:53.127Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:23:53.228Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3361354"} 2022-11-23T13:23:53.228Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-23T13:23:53.232Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site3":{"state":"InProgress","policyIndex":1}}, "isBatchComplete": false} 2022-11-23T13:23:53.245Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueAfter": 300} 2022-11-23T13:28:33.138Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:28:33.238Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3361354"} 2022-11-23T13:28:33.238Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-23T13:28:33.242Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site3":{"state":"Completed"}}, "isBatchComplete": true} 2022-11-23T13:28:33.248Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": true} 2022-11-23T13:28:33.253Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:28:33.354Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3379412"} 2022-11-23T13:28:33.354Z INFO controllers.ClusterGroupUpgrade Upgrade is completed 2022-11-23T13:28:33.401Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": false} 2022-11-23T13:28:33.401Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-install/ztp-sriov"} 2022-11-23T13:28:33.501Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-install/ztp-sriov", "version": "3379435"} 2022-11-23T13:28:33.502Z INFO controllers.ClusterGroupUpgrade Upgrade is completed 2022-11-23T13:28:33.508Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-install/ztp-sriov", "requeueRightAway": false} 2022-11-28T20:46:36.232Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-policies/changes3"} 2022-11-28T20:46:36.333Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-policies/changes3", "version": "30833223"} 2022-11-28T20:46:36.338Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-policies/changes3", "requeueRightAway": true} 2022-11-28T20:46:36.338Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-policies/changes3"} 2022-11-28T20:46:36.439Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-policies/changes3", "version": "30833229"} 2022-11-28T20:46:36.445Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-policies/changes3", "requeueRightAway": true} 2022-11-28T20:46:36.445Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-policies/changes3"} 2022-11-28T20:46:36.545Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-policies/changes3", "version": "30833242"} 2022-11-28T20:46:36.552Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-policies/changes3", "requeueRightAway": true} 2022-11-28T20:46:36.552Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-policies/changes3"} 2022-11-28T20:46:36.653Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-policies/changes3", "version": "30833248"} 2022-11-28T20:46:36.653Z INFO controllers.ClusterGroupUpgrade [doManagedPoliciesExist] {"policyMap": {"cloudransno-site1-perfprofile-policy":"ztp-policies","cloudransno-site1-ptp-config-policy":"ztp-policies","cloudransno-site1-sriov-dpdk-policy":"ztp-policies","cloudransno-site1-sriov-nw-dpdk-policy":"ztp-policies","cloudransno-site1-tuned-perf-patch-policy":"ztp-policies","common-kdump-policy":"ztp-common","common-mon-offload-policy":"ztp-common","common-pao-sub-policy":"ztp-common","common-ptp-sub-policy":"ztp-common","common-sriov-sub-policy":"ztp-common","common-storage-sub-policy":"ztp-common","common-subscriptions-policy":"ztp-common","group-du-sno-mc-sctp-policy":"ztp-group-du-sno"}} 2022-11-28T20:46:36.657Z INFO controllers.ClusterGroupUpgrade [getClustersNonCompliantWithPolicy] {"policy: ": "cloudransno-site1-sriov-dpdk-policy", "clusters: ": ["cloudransno-site1"]} 2022-11-28T20:46:36.657Z INFO controllers.ClusterGroupUpgrade Remediation plan {"remediatePlan": [["cloudransno-site1"]]} 2022-11-28T20:46:36.727Z INFO controllers.ClusterGroupUpgrade [getPolicyContent] Policy spec.policy-templates.objectDefinition.spec.object-templates.kind is not of Subscription kind {"policyName": "cloudransno-site1-sriov-dpdk-policy"} 2022-11-28T20:46:36.727Z INFO controllers.ClusterGroupUpgrade [blockingCRsNotCompleted] {"blockingCRs": []} 2022-11-28T20:46:36.732Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-policies/changes3", "requeueRightAway": true} 2022-11-28T20:46:36.732Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-policies/changes3"} 2022-11-28T20:46:36.833Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-policies/changes3", "version": "30833265"} 2022-11-28T20:46:36.833Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 0} 2022-11-28T20:46:36.833Z INFO controllers.ClusterGroupUpgrade [initializeRemediationPolicyForBatch] {"CurrentBatchRemediationProgress": {"cloudransno-site1":{"state":"NotStarted"}}} 2022-11-28T20:46:36.836Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site1":{"state":"InProgress","policyIndex":0}}, "isBatchComplete": false} 2022-11-28T20:46:36.851Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-policies/changes3", "requeueRightAway": true} 2022-11-28T20:46:36.851Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-policies/changes3"} 2022-11-28T20:46:36.952Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-policies/changes3", "version": "30833275"} 2022-11-28T20:46:36.952Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-28T20:46:36.956Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site1":{"state":"InProgress","policyIndex":0}}, "isBatchComplete": false} 2022-11-28T20:46:36.969Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-policies/changes3", "requeueAfter": 299.047420579} 2022-11-28T20:46:36.969Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-policies/changes3"} 2022-11-28T20:46:37.070Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-policies/changes3", "version": "30833275"} 2022-11-28T20:46:37.070Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-28T20:46:37.073Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site1":{"state":"InProgress","policyIndex":0}}, "isBatchComplete": false} 2022-11-28T20:46:37.088Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-policies/changes3", "requeueAfter": 298.92972846} 2022-11-28T20:46:53.099Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-policies/changes3"} 2022-11-28T20:46:53.200Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-policies/changes3", "version": "30833275"} 2022-11-28T20:46:53.200Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-28T20:46:53.205Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site1":{"state":"InProgress","policyIndex":0}}, "isBatchComplete": false} 2022-11-28T20:46:53.218Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-policies/changes3", "requeueAfter": 282.799601645} 2022-11-28T20:51:36.017Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-policies/changes3"} 2022-11-28T20:51:36.118Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-policies/changes3", "version": "30833275"} 2022-11-28T20:51:36.118Z INFO controllers.ClusterGroupUpgrade [Reconcile] {"Status.CurrentBatch": 1} 2022-11-28T20:51:36.121Z INFO controllers.ClusterGroupUpgrade [getNextRemediationPoliciesForBatch] {"plan": {"cloudransno-site1":{"state":"Completed"}}, "isBatchComplete": true} 2022-11-28T20:51:36.127Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-policies/changes3", "requeueRightAway": true} 2022-11-28T20:51:36.133Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-policies/changes3"} 2022-11-28T20:51:36.233Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-policies/changes3", "version": "30851320"} 2022-11-28T20:51:36.233Z INFO controllers.ClusterGroupUpgrade Upgrade is completed 2022-11-28T20:51:36.266Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-policies/changes3", "requeueRightAway": false} 2022-11-28T20:51:36.266Z INFO controllers.ClusterGroupUpgrade Start reconciling CGU {"name": "ztp-policies/changes3"} 2022-11-28T20:51:36.366Z INFO controllers.ClusterGroupUpgrade Loaded CGU {"name": "ztp-policies/changes3", "version": "30851330"} 2022-11-28T20:51:36.366Z INFO controllers.ClusterGroupUpgrade Upgrade is completed 2022-11-28T20:51:36.372Z INFO controllers.ClusterGroupUpgrade Finish reconciling CGU {"name": "ztp-policies/changes3", "requeueRightAway": false}