windows-machine-config-operator-container-10.17.0-29
10.17.0-a9ab900
4.17.0-0.nightly-2024-09-13-193731

1. install 4.16 nightly with wmco 10.16.1 on AWS 2019
2. Upgrade OCP 4.16 to 4.17 (oc adm upgrade --to-image=registry.ci.openshift.org/ocp/release:4.17.0-0.nightly-2024-09-13-193731 --allow-explicit-upgrade=true --force --allow-not-recommended --allow-upgrade-with-warnings)
3. Upgrade wmco to 10.17.0

Issue does not reproduce on aws 2022

wmco log:
"level":"info","ts":"2024-09-18T04:57:24Z","logger":"controllers.registry","msg":"updating containerd config","registry":{"name":""},"directory":"C:\\k\\containerd\\registries","node":"ip-10-0-9-9.us-east-2.compute.internal"}
{"level":"error","ts":"2024-09-18T04:57:24Z","msg":"Reconciler error","controller":"imagedigestmirrorset","controllerGroup":"config.openshift.io","controllerKind":"ImageDigestMirrorSet","ImageDigestMirrorSet":{"name":""},"namespace":"","name":"","reconcileID":"c3d71406-772d-4641-b0fa-b69c6afa99d1","error":"unable to remove directory C:\\k\\containerd\\registries, out: , err: error running powershell.exe -NonInteractive -ExecutionPolicy Bypass \"if(Test-Path C:\\k\\containerd\\registries) {Remove-Item -Recurse -Force C:\\k\\containerd\\registries}\": read tcp 10.0.61.13:45914->10.0.9.9:22: read: connection reset by peer","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:222"}
{"level":"info","ts":"2024-09-18T04:57:24Z","logger":"controller.windowsmachine","msg":"deleting machine","windowsmachine":{"name":"weinliu-3039-b5xdn-windows-worker-us-east-2a-w48r4","namespace":"openshift-machine-api"}}
{"level":"info","ts":"2024-09-18T04:57:24Z","logger":"controller.windowsmachine","msg":"unhealthy machine count for machineset","name":"weinliu-3039-b5xdn-windows-worker-us-east-2a","total":2,"unhealthy":1}
{"level":"info","ts":"2024-09-18T04:57:24Z","logger":"controller.windowsmachine","msg":"machine deletion restricted","windowsmachine":{"name":"weinliu-3039-b5xdn-windows-worker-us-east-2a-w48r4","namespace":"openshift-machine-api"},"maxUnhealthyCount":1}
{"level":"info","ts":"2024-09-18T04:57:26Z","logger":"controller.windowsmachine","msg":"deleting machine","windowsmachine":{"name":"weinliu-3039-b5xdn-windows-worker-us-east-2a-w48r4","namespace":"openshift-machine-api"}}
{"level":"info","ts":"2024-09-18T04:57:26Z","logger":"controller.windowsmachine","msg":"unhealthy machine count for machineset","name":"weinliu-3039-b5xdn-windows-worker-us-east-2a","total":2,"unhealthy":1}
{"level":"info","ts":"2024-09-18T04:57:26Z","logger":"controller.windowsmachine","msg":"machine deletion restricted","windowsmachine":{"name":"weinliu-3039-b5xdn-windows-worker-us-east-2a-w48r4","namespace":"openshift-machine-api"},"maxUnhealthyCount":1}
{"level":"info","ts":"2024-09-18T04:57:28Z","logger":"controller.windowsmachine","msg":"deleting machine","windowsmachine":{"name":"weinliu-3039-b5xdn-windows-worker-us-east-2a-w48r4","namespace":"openshift-machine-api"}}
{"level":"info","ts":"2024-09-18T04:57:28Z","logger":"controllers.configmap","msg":"instance requires upgrade","node":"ip-10-0-18-131.us-east-2.compute.internal","version":"10.16.1-b5739a7","expected version":"10.17.0-a9ab900"}
{"level":"info","ts":"2024-09-18T04:57:28Z","logger":"nc 10.0.18.131","msg":"deconfiguring"}
{"level":"info","ts":"2024-09-18T04:57:28Z","logger":"controller.windowsmachine","msg":"unhealthy machine count for machineset","name":"weinliu-3039-b5xdn-windows-worker-us-east-2a","total":2,"unhealthy":1}
{"level":"info","ts":"2024-09-18T04:57:28Z","logger":"controller.windowsmachine","msg":"machine deletion restricted","windowsmachine":{"name":"weinliu-3039-b5xdn-windows-worker-us-east-2a-w48r4","namespace":"openshift-machine-api"},"maxUnhealthyCount":1}
{"level":"info","ts":"2024-09-18T04:57:29Z","logger":"wc 10.0.18.131","msg":"deconfigured","service":"windows-instance-config-daemon"}
{"level":"info","ts":"2024-09-18T04:57:30Z","logger":"controller.windowsmachine","msg":"deleting machine","windowsmachine":{"name":"weinliu-3039-b5xdn-windows-worker-us-east-2a-w48r4","namespace":"openshift-machine-api"}}
{"level":"info","ts":"2024-09-18T04:57:31Z","logger":"controller.windowsmachine","msg":"unhealthy machine count for machineset","name":"weinliu-3039-b5xdn-windows-worker-us-east-2a","total":2,"unhealthy":1}
{"level":"info","ts":"2024-09-18T04:57:31Z","logger":"controller.windowsmachine","msg":"machine deletion restricted","windowsmachine":{"name":"weinliu-3039-b5xdn-windows-worker-us-east-2a-w48r4","namespace":"openshift-machine-api"},"maxUnhealthyCount":1}
{"level":"error","ts":"2024-09-18T04:57:32Z","msg":"Reconciler error","controller":"configmap","controllerGroup":"","controllerKind":"ConfigMap","ConfigMap":{"name":"windows-instances","namespace":"openshift-windows-machine-config-operator"},"namespace":"openshift-windows-machine-config-operator","name":"windows-instances","reconcileID":"fbb75d9b-a114-46b5-a44b-3d2e7c34cf9b","error":"error configuring host with address 10.0.18.131: unable to cleanup the Windows instance: error copying /payload/windows-instance-config-daemon.exe to C:\\k: unable to transfer /payload/windows-instance-config-daemon.exe to remote dir C:\\k: error initializing C:\\k\\windows-instance-config-daemon.exe file on Windows VM: sftp: \"Failure\" (SSH_FX_FAILURE)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:222"}
{"level":"info","ts":"2024-09-18T04:57:32Z","logger":"controllers.configmap","msg":"processing","instances in":"windows-instances"}
{"level":"info","ts":"2024-09-18T04:57:33Z","logger":"controllers.controllerconfig","msg":"updating kubelet CA client certificates in","node":"ip-10-0-18-131.us-east-2.compute.internal"}
{"level":"info","ts":"2024-09-18T04:57:34Z","logger":"controllers.registry","msg":"updating containerd config","registry":{"name":""},"registry":{"name":""},"directory":"C:\\k\\containerd\\registries","node":"ip-10-0-27-91.us-east-2.compute.internal"}
{"level":"info","ts":"2024-09-18T04:57:36Z","logger":"controller.windowsmachine","msg":"deleting machine","windowsmachine":{"name":"weinliu-3039-b5xdn-windows-worker-us-east-2a-w48r4","namespace":"openshift-machine-api"}}
{"level":"info","ts":"2024-09-18T04:57:36Z","logger":"controller.windowsmachine","msg":"unhealthy machine count for machineset","name":"weinliu-3039-b5xdn-windows-worker-us-east-2a","total":2,"unhealthy":1}
{"level":"info","ts":"2024-09-18T04:57:36Z","logger":"controller.windowsmachine","msg":"machine deletion restricted","windowsmachine":{"name":"weinliu-3039-b5xdn-windows-worker-us-east-2a-w48r4","namespace":"openshift-machine-api"},"maxUnhealthyCount":1}
{"level":"info","ts":"2024-09-18T04:57:38Z","logger":"controllers.configmap","msg":"instance requires upgrade","node":"ip-10-0-18-131.us-east-2.compute.internal","version":"10.16.1-b5739a7","expected version":"10.17.0-a9ab900"}
{"level":"info","ts":"2024-09-18T04:57:38Z","logger":"nc 10.0.18.131","msg":"deconfiguring"}
{"level":"error","ts":"2024-09-18T04:57:40Z","msg":"Reconciler error","controller":"configmap","controllerGroup":"","controllerKind":"ConfigMap","ConfigMap":{"name":"windows-instances","namespace":"openshift-windows-machine-config-operator"},"namespace":"openshift-windows-machine-config-operator","name":"windows-instances","reconcileID":"5a36bacb-617b-4b35-b852-e209a76d3b97","error":"error configuring host with address 10.0.18.131: unable to cleanup the Windows instance: error copying /payload/windows-instance-config-daemon.exe to C:\\k: unable to transfer /payload/windows-instance-config-daemon.exe to remote dir C:\\k: error initializing C:\\k\\windows-instance-config-daemon.exe file on Windows VM: sftp: \"Failure\" (SSH_FX_FAILURE)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:222"}
{"level":"info","ts":"2024-09-18T04:57:40Z","logger":"controllers.configmap","msg":"processing","instances in":"windows-instances"}
{"level":"info","ts":"2024-09-18T04:57:43Z","logger":"controllers.configmap","msg":"instance requires upgrade","node":"ip-10-0-18-131.us-east-2.compute.internal","version":"10.16.1-b5739a7","expected version":"10.17.0-a9ab900"}
{"level":"info","ts":"2024-09-18T04:57:43Z","logger":"nc 10.0.18.131","msg":"deconfiguring"}
{"level":"error","ts":"2024-09-18T04:57:45Z","msg":"Reconciler error","controller":"configmap","controllerGroup":"","controllerKind":"ConfigMap","ConfigMap":{"name":"windows-instances","namespace":"openshift-windows-machine-config-operator"},"namespace":"openshift-windows-machine-config-operator","name":"windows-instances","reconcileID":"b0732dc8-fe7a-4251-915e-7595d2b8b5f4","error":"error configuring host with address 10.0.18.131: unable to cleanup the Windows instance: error copying /payload/windows-instance-config-daemon.exe to C:\\k: unable to transfer /payload/windows-instance-config-daemon.exe to remote dir C:\\k: error initializing C:\\k\\windows-instance-config-daemon.exe file on Windows VM: sftp: \"Failure\" (SSH_FX_FAILURE)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:222"}
{"level":"info","ts":"2024-09-18T04:57:45Z","logger":"controllers.configmap","msg":"processing","instances in":"windows-instances"}
{"level":"info","ts":"2024-09-18T04:57:46Z","logger":"controller.windowsmachine","msg":"deleting machine","windowsmachine":{"name":"weinliu-3039-b5xdn-windows-worker-us-east-2a-w48r4","namespace":"openshift-machine-api"}}
{"level":"info","ts":"2024-09-18T04:57:46Z","logger":"controller.windowsmachine","msg":"unhealthy machine count for machineset","name":"weinliu-3039-b5xdn-windows-worker-us-east-2a","total":2,"unhealthy":1}
{"level":"info","ts":"2024-09-18T04:57:46Z","logger":"controller.windowsmachine","msg":"machine deletion restricted","windowsmachine":{"name":"weinliu-3039-b5xdn-windows-worker-us-east-2a-w48r4","namespace":"openshift-machine-api"},"maxUnhealthyCount":1}
{"level":"info","ts":"2024-09-18T04:57:48Z","logger":"controllers.configmap","msg":"instance requires upgrade","node":"ip-10-0-18-131.us-east-2.compute.internal","version":"10.16.1-b5739a7","expected version":"10.17.0-a9ab900"}
{"level":"info","ts":"2024-09-18T04:57:48Z","logger":"nc 10.0.18.131","msg":"deconfiguring"}
{"level":"error","ts":"2024-09-18T04:57:50Z","msg":"Reconciler error","controller":"configmap","controllerGroup":"","controllerKind":"ConfigMap","ConfigMap":{"name":"windows-instances","namespace":"openshift-windows-machine-config-operator"},"namespace":"openshift-windows-machine-config-operator","name":"windows-instances","reconcileID":"377264a3-800f-40e3-b4b6-3261f85e5015","error":"error configuring host with address 10.0.18.131: unable to cleanup the Windows instance: error copying /payload/windows-instance-config-daemon.exe to C:\\k: unable to transfer /payload/windows-instance-config-daemon.exe to remote dir C:\\k: error initializing C:\\k\\windows-instance-config-daemon.exe file on Windows VM: sftp: \"Failure\" (SSH_FX_FAILURE)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:222"}
{"level":"info","ts":"2024-09-18T04:57:50Z","logger":"controllers.configmap","msg":"processing","instances in":"windows-instances"}
{"level":"info","ts":"2024-09-18T04:57:51Z","logger":"controller.windowsmachine","msg":"processing","windowsmachine":{"name":"weinliu-3039-b5xdn-windows-worker-us-east-2a-mwk29","namespace":"openshift-machine-api"},"address":"10.0.22.88"}
{"level":"info","ts":"2024-09-18T04:57:52Z","logger":"controllers.configmap","msg":"instance requires upgrade","node":"ip-10-0-18-131.us-east-2.compute.internal","version":"10.16.1-b5739a7","expected version":"10.17.0-a9ab900"}
{"level":"info","ts":"2024-09-18T04:57:52Z","logger":"nc 10.0.18.131","msg":"deconfiguring"}
{"level":"error","ts":"2024-09-18T04:57:54Z","msg":"Reconciler error","controller":"configmap","controllerGroup":"","controllerKind":"ConfigMap","ConfigMap":{"name":"windows-instances","namespace":"openshift-windows-machine-config-operator"},"namespace":"openshift-windows-machine-config-operator","name":"windows-instances","reconcileID":"ff237e5b-f9aa-4df7-b60a-5bc9fb4e0b8b","error":"error configuring host with address 10.0.18.131: unable to cleanup the Windows instance: error copying /payload/windows-instance-config-daemon.exe to C:\\k: unable to transfer /payload/windows-instance-config-daemon.exe to remote dir C:\\k: error initializing C:\\k\\windows-instance-config-daemon.exe file on Windows VM: sftp: \"Failure\" (SSH_FX_FAILURE)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:222"}
{"level":"info","ts":"2024-09-18T04:57:54Z","logger":"controllers.configmap","msg":"processing","instances in":"windows-instances"}
{"level":"info","ts":"2024-09-18T04:57:56Z","logger":"controllers.configmap","msg":"instance requires upgrade","node":"ip-10-0-18-131.us-east-2.compute.internal","version":"10.16.1-b5739a7","expected version":"10.17.0-a9ab900"}
{"level":"info","ts":"2024-09-18T04:57:56Z","logger":"nc 10.0.18.131","msg":"deconfiguring"}
{"level":"error","ts":"2024-09-18T04:57:58Z","msg":"Reconciler error","controller":"configmap","controllerGroup":"","controllerKind":"ConfigMap","ConfigMap":{"name":"windows-instances","namespace":"openshift-windows-machine-config-operator"},"namespace":"openshift-windows-machine-config-operator","name":"windows-instances","reconcileID":"a831f9d9-14cc-41c8-b521-28251e658b58","error":"error configuring host with address 10.0.18.131: unable to cleanup the Windows instance: error copying /payload/windows-instance-config-daemon.exe to C:\\k: unable to transfer /payload/windows-instance-config-daemon.exe to remote dir C:\\k: error initializing C:\\k\\windows-instance-config-daemon.exe file on Windows VM: sftp: \"Failure\" (SSH_FX_FAILURE)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:222"}
{"level":"info","ts":"2024-09-18T04:57:58Z","logger":"controllers.configmap","msg":"processing","instances in":"windows-instances"}
{"level":"info","ts":"2024-09-18T04:58:00Z","logger":"controllers.configmap","msg":"instance requires upgrade","node":"ip-10-0-18-131.us-east-2.compute.internal","version":"10.16.1-b5739a7","expected version":"10.17.0-a9ab900"}
{"level":"info","ts":"2024-09-18T04:58:00Z","logger":"nc 10.0.18.131","msg":"deconfiguring"}
{"level":"info","ts":"2024-09-18T04:59:39Z","logger":"wc 10.0.18.131","msg":"rebooting instance"}
{"level":"info","ts":"2024-09-18T05:01:18Z","logger":"wc 10.0.22.88","msg":"configuring"}
{"level":"info","ts":"2024-09-18T05:02:24Z","logger":"wc 10.0.22.88","msg":"rebooting instance"}
{"level":"info","ts":"2024-09-18T05:02:48Z","logger":"wc 10.0.22.88","msg":"transferring files"}
{"level":"info","ts":"2024-09-18T05:05:04Z","logger":"wc 10.0.22.88","msg":"configured","service":"windows-instance-config-daemon","args":"controller --windows-service --log-dir C:\\var\\log\\wicd --kubeconfig C:\\k\\wicd-kubeconfig --namespace openshift-windows-machine-config-operator --ca-bundle C:\\k\\ca-bundle.crt"}
{"level":"info","ts":"2024-09-18T05:06:19Z","logger":"nc 10.0.22.88","msg":"instance has been configured as a worker node","version":"10.17.0-a9ab900"}
{"level":"info","ts":"2024-09-18T05:06:19Z","logger":"metrics","msg":"Prometheus configured","endpoints":"windows-exporter","port":9182,"name":"metrics"}
{"level":"info","ts":"2024-09-18T05:06:19Z","logger":"controller.windowsmachine","msg":"deleting machine","windowsmachine":{"name":"weinliu-3039-b5xdn-windows-worker-us-east-2a-w48r4","namespace":"openshift-machine-api"}}
{"level":"info","ts":"2024-09-18T05:06:19Z","logger":"controller.windowsmachine","msg":"unhealthy machine count for machineset","name":"weinliu-3039-b5xdn-windows-worker-us-east-2a","total":2,"unhealthy":0}
{"level":"info","ts":"2024-09-18T05:06:19Z","logger":"controller.windowsmachine","msg":"machine has been remediated by deletion","name":"weinliu-3039-b5xdn-windows-worker-us-east-2a-w48r4"}
{"level":"info","ts":"2024-09-18T05:06:19Z","logger":"metrics","msg":"Prometheus configured","endpoints":"windows-exporter","port":9182,"name":"metrics"}
{"level":"info","ts":"2024-09-18T05:06:53Z","logger":"controller.windowsmachine","msg":"processing","windowsmachine":{"name":"weinliu-3039-b5xdn-windows-worker-us-east-2a-6hnpt","namespace":"openshift-machine-api"},"address":"10.0.17.138"}
{"level":"error","ts":"2024-09-18T05:09:38Z","msg":"Reconciler error","controller":"configmap","controllerGroup":"","controllerKind":"ConfigMap","ConfigMap":{"name":"windows-instances","namespace":"openshift-windows-machine-config-operator"},"namespace":"openshift-windows-machine-config-operator","name":"windows-instances","reconcileID":"a2e844d5-0b15-47de-b8bc-2eb65e1c4520","error":"error configuring host with address 10.0.18.131: timeout waiting for windowsmachineconfig.openshift.io/reboot-required to be cleared: timed out waiting for the condition","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:222"}
{"level":"info","ts":"2024-09-18T05:09:38Z","logger":"controllers.configmap","msg":"processing","instances in":"windows-instances"}
{"level":"info","ts":"2024-09-18T05:09:40Z","logger":"controllers.configmap","msg":"instance requires upgrade","node":"ip-10-0-27-91.us-east-2.compute.internal","version":"10.16.1-b5739a7","expected version":"10.17.0-a9ab900"}
{"level":"error","ts":"2024-09-18T05:09:40Z","msg":"Reconciler error","controller":"configmap","controllerGroup":"","controllerKind":"ConfigMap","ConfigMap":{"name":"windows-instances","namespace":"openshift-windows-machine-config-operator"},"namespace":"openshift-windows-machine-config-operator","name":"windows-instances","reconcileID":"86d374d2-158b-4434-b2ee-595d033e2e48","error":"error configuring host with address 10.0.27.91: cannot mark node ip-10-0-27-91.us-east-2.compute.internal as upgrading, maximum number of parallel upgrading nodes reached (1)","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:222"}
{"level":"info","ts":"2024-09-18T05:09:40Z","logger":"controllers.configmap","msg":"processing","instances in":"windows-instances"}
{"level":"info","ts":"2024-09-18T05:10:29Z","logger":"wc 10.0.17.138","msg":"configuring"}
{"level":"info","ts":"2024-09-18T05:11:45Z","logger":"wc 10.0.17.138","msg":"rebooting instance"}
{"level":"info","ts":"2024-09-18T05:12:07Z","logger":"wc 10.0.17.138","msg":"transferring files"}
{"level":"error","ts":"2024-09-18T05:14:05Z","msg":"Reconciler error","controller":"imagedigestmirrorset","controllerGroup":"config.openshift.io","controllerKind":"ImageDigestMirrorSet","ImageDigestMirrorSet":{"name":""},"namespace":"","name":"","reconcileID":"98816b9a-d12a-4816-bfbb-7f7095afa6f5","error":"failed to create new nodeconfig: error instantiating Windows instance from VM: unable to setup VM 10.0.9.9 sshConnectivity: error instantiating SSH client: unable to connect to Windows VM 10.0.9.9: timed out waiting for the condition","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:222"}
{"level":"info","ts":"2024-09-18T05:14:07Z","logger":"controllers.registry","msg":"updating containerd config","registry":{"name":""},"registry":{"name":""},"registry":{"name":""},"directory":"C:\\k\\containerd\\registries","node":"ip-10-0-27-91.us-east-2.compute.internal"}
{"level":"info","ts":"2024-09-18T05:14:10Z","logger":"controllers.registry","msg":"updating containerd config","registry":{"name":""},"registry":{"name":""},"registry":{"name":""},"directory":"C:\\k\\containerd\\registries","node":"ip-10-0-22-88.us-east-2.compute.internal"}
{"level":"error","ts":"2024-09-18T05:14:13Z","msg":"Reconciler error","controller":"secret","controllerGroup":"","controllerKind":"Secret","Secret":{"name":"windows-machine-config-operator-tls","namespace":"openshift-windows-machine-config-operator"},"namespace":"openshift-windows-machine-config-operator","name":"windows-machine-config-operator-tls","reconcileID":"4c98e4ea-34bc-4c01-b288-e6f7e748860a","error":"failed to create new nodeconfig: error instantiating Windows instance from VM: unable to setup VM 10.0.9.9 sshConnectivity: error instantiating SSH client: unable to connect to Windows VM 10.0.9.9: timed out waiting for the condition","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:222"}
{"level":"error","ts":"2024-09-18T05:14:13Z","msg":"Reconciler error","controller":"secret","controllerGroup":"","controllerKind":"Secret","Secret":{"name":"windows-machine-config-operator-tls","namespace":"openshift-windows-machine-config-operator"},"namespace":"openshift-windows-machine-config-operator","name":"windows-machine-config-operator-tls","reconcileID":"cc3039c6-29d8-4669-8a92-4eb109425987","error":"unable to create instance object from node: node is missing valid username annotation","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:222"}
{"level":"error","ts":"2024-09-18T05:14:21Z","msg":"Reconciler error","controller":"secret","controllerGroup":"","controllerKind":"Secret","Secret":{"name":"windows-machine-config-operator-tls","namespace":"openshift-windows-machine-config-operator"},"namespace":"openshift-windows-machine-config-operator","name":"windows-machine-config-operator-tls","reconcileID":"5c9cc6a3-3d44-4107-83ab-d3e32e3cb503","error":"unable to create instance object from node: node is missing valid username annotation","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:222"}
{"level":"info","ts":"2024-09-18T05:14:28Z","logger":"wc 10.0.17.138","msg":"configured","service":"windows-instance-config-daemon","args":"controller --windows-service --log-dir C:\\var\\log\\wicd --kubeconfig C:\\k\\wicd-kubeconfig --namespace openshift-windows-machine-config-operator --ca-bundle C:\\k\\ca-bundle.crt"}
{"level":"error","ts":"2024-09-18T05:16:12Z","msg":"Reconciler error","controller":"node","controllerGroup":"","controllerKind":"Node","Node":{"name":"ip-10-0-18-131.us-east-2.compute.internal"},"namespace":"","name":"ip-10-0-18-131.us-east-2.compute.internal","reconcileID":"d5de6fc5-0473-4c56-88de-45cb03b90dec","error":"full instance reboot failed: error reinitializing SSH connection after VM reboot: failed to reinitialize ssh client: unable to connect to Windows VM 10.0.18.131: timed out waiting for the condition","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:324\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:261\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2\n\t/remote-source/build/windows-machine-config-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:222"}
{"level":"info","ts":"2024-09-18T05:16:14Z","logger":"nc 10.0.22.88","msg":"evicting pod winc-test/win-webserver-64d999dd59-d8bfg\n"}
{"level":"info","ts":"2024-09-18T05:16:14Z","logger":"nc 10.0.22.88","msg":"evicting pod winc-test/win-webserver-64d999dd59-26klm\n"}