Uploaded image for project: 'Red Hat Advanced Cluster Management'
  1. Red Hat Advanced Cluster Management
  2. ACM-14965

ClusterInstance condition - RenderedTemplatesValidated with message "Rendered manifests failed dry-run validation" however cluster successfully provisioned anyway

XMLWordPrintable

    • False
    • None
    • False
    • None

      Description of problem:

      While provisioning 297 SNOs concurrently with clusterinstance using IBI, 2 clusters vm00007 and vm00011 both have conditions which contain a message claiming that dry-run failed however the two clusters both completed provisioning entirely.

      The condition in question is:

        - lastTransitionTime: "2024-10-16T03:18:31Z"
    message: Rendered manifests failed dry-run validation
    reason: Failed
    status: "False"
    type: RenderedTemplatesValidated

      Version-Release number of selected component (if applicable):

      OCP Hub and IBI deployed clusters 4.17.1

      ACM 2.12 - 2.12.0-DOWNSTREAM-2024-10-10-17-47-25

      How reproducible:

      Steps to Reproduce:

      1.  
      2.  
      3. ...

      Actual results:

      Expected results:

      Additional info:

      mplete yaml manifest for vm00007:

       

      # oc get clusterinstance -n vm00007 vm00007 -o yaml
apiVersion: siteconfig.open-cluster-management.io/v1alpha1
kind: ClusterInstance
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"siteconfig.open-cluster-management.io/v1alpha1","kind":"ClusterInstance","metadata":{"annotations":{},"labels":{"app.kubernetes.io/instance":"ztp-clusters-01"},"name":"vm00007","namespace":"vm00007"},"spec":{"additionalNTPSources":["fc00:1005::21"],"baseDomain":"rdu2.scalelab.redhat.com","clusterImageSetNameRef":"openshift-4.17.1","clusterName":"vm00007","clusterNetwork":[{"cidr":"fd01::/48","hostPrefix":64}],"clusterType":"SNO","extraLabels":{"ManagedCluster":{"common":"true","du-profile":"4.17","group-du-sno":"","name":"vm00007","observability":"enabled","sites":"vm00007","vendor":"OpenShift"}},"holdInstallation":false,"installConfigOverrides":"{\"capabilities\":{\"baselineCapabilitySet\": \"None\", \"additionalEnabledCapabilities\": [ \"OperatorLifecycleManager\", \"Ingress\", \"NodeTuning\" ] }}","machineNetwork":[{"cidr":"fc00:1005::/64"}],"networkType":"OVNKubernetes","nodes":[{"automatedCleaningMode":"disabled","bmcAddress":"redfish-virtualmedia+http://[fc00:1005::21]:9000/redfish/v1/Systems/54690a27-5069-5f72-9bad-ca66e6097ee0","bmcCredentialsName":{"name":"vm00007-bmc-secret"},"bootMACAddress":"52:54:00:00:00:07","bootMode":"UEFI","hostName":"vm00007","ironicInspect":"disabled","nodeNetwork":{"config":{"dns-resolver":{"config":{"server":["fc00:1005::21"]}},"interfaces":[{"identifier":"mac-address","ipv4":{"enabled":false},"ipv6":{"address":[{"ip":"fc00:1005::3ee","prefix-length":64}],"autoconf":false,"dhcp":false,"enabled":true},"mac-address":"52:54:00:00:00:07","name":"enp1s0","state":"up","type":"ethernet"}],"routes":{"config":[{"destination":"::/0","next-hop-address":"fc00:1005::1","next-hop-interface":"enp1s0"}]}},"interfaces":[{"macAddress":"52:54:00:00:00:07","name":"enp1s0"}]},"templateRefs":[{"name":"ibi-node-templates-v1","namespace":"open-cluster-management"}]}],"pullSecretRef":{"name":"ibi-pull-secret"},"serviceNetwork":[{"cidr":"fd02::/112"}],"sshPublicKey":"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6351YHGIvE6DZcAt0RXqodzKbglUeCNqxRyd7OOfcN+p88RaKZahg9/dL81IbdlGPbhPPG9ga7BdkLN8VbyYNCjE7kBIKM47JS1pbjeoJhI8bBrFjjp6LIJW/tWh1Pyl7Mk3DbPiZOkJ9HXvOgE/HRh44jJPtMLnzrZU5VNgNRgaEBWOG+j06pxdK9giMji1mFkJXSr43YUZYYgM3egNfNzxeTG0SshbZarRDEKeAlnDJkZ70rbP2krL2MgZJDv8vIK1PcMMFhjsJ/4Pp7F0Tl2Rm/qlZhTn4ptWagZmM0Z3N2WkNdX6Z9i2lZ5K+5jNHEFfjw/CPOFqpaFMMckpfFMsAJchbqnh+F5NvKJSFNB6L77iRCp5hbhGBbZncwc3UDO3FZ9ZuYZ8Ws+2ZyS5uVxd5ZUsvZFO+mWwySytFbsc0nUUcgkXlBiGKF/eFm9SQTURkyNzJkJfPm7awRwYoidaf8MTSp/kUCCyloAjpFIOJAa0SoVerhLp8uhQzfeU= root@e38-h01-000-r650.rdu2.scalelab.redhat.com","templateRefs":[{"name":"ibi-cluster-templates-v1","namespace":"open-cluster-management"}]}}
  creationTimestamp: "2024-10-16T03:16:45Z"
  finalizers:
  - clusterinstance.siteconfig.open-cluster-management.io/finalizer
  generation: 1
  labels:
    app.kubernetes.io/instance: ztp-clusters-01
  name: vm00007
  namespace: vm00007
  resourceVersion: "1804043"
  uid: 230641e0-aab8-48a1-ac2a-7d2d7ecaf342
spec:
  additionalNTPSources:
  - fc00:1005::21
  baseDomain: rdu2.scalelab.redhat.com
  clusterImageSetNameRef: openshift-4.17.1
  clusterName: vm00007
  clusterNetwork:
  - cidr: fd01::/48
    hostPrefix: 64
  clusterType: SNO
  cpuPartitioningMode: None
  extraLabels:
    ManagedCluster:
      common: "true"
      du-profile: "4.17"
      group-du-sno: ""
      name: vm00007
      observability: enabled
      sites: vm00007
      vendor: OpenShift
  holdInstallation: false
  installConfigOverrides: '{"capabilities":{"baselineCapabilitySet": "None", "additionalEnabledCapabilities":
    [ "OperatorLifecycleManager", "Ingress", "NodeTuning" ] }}'
  machineNetwork:
  - cidr: fc00:1005::/64
  networkType: OVNKubernetes
  nodes:
  - automatedCleaningMode: disabled
    bmcAddress: redfish-virtualmedia+http://[fc00:1005::21]:9000/redfish/v1/Systems/54690a27-5069-5f72-9bad-ca66e6097ee0
    bmcCredentialsName:
      name: vm00007-bmc-secret
    bootMACAddress: "52:54:00:00:00:07"
    bootMode: UEFI
    hostName: vm00007
    ironicInspect: disabled
    nodeNetwork:
      config:
        dns-resolver:
          config:
            server:
            - fc00:1005::21
        interfaces:
        - identifier: mac-address
          ipv4:
            enabled: false
          ipv6:
            address:
            - ip: fc00:1005::3ee
              prefix-length: 64
            autoconf: false
            dhcp: false
            enabled: true
          mac-address: "52:54:00:00:00:07"
          name: enp1s0
          state: up
          type: ethernet
        routes:
          config:
          - destination: ::/0
            next-hop-address: fc00:1005::1
            next-hop-interface: enp1s0
      interfaces:
      - macAddress: "52:54:00:00:00:07"
        name: enp1s0
    role: master
    templateRefs:
    - name: ibi-node-templates-v1
      namespace: open-cluster-management
  pullSecretRef:
    name: ibi-pull-secret
  serviceNetwork:
  - cidr: fd02::/112
  sshPublicKey: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6351YHGIvE6DZcAt0RXqodzKbglUeCNqxRyd7OOfcN+p88RaKZahg9/dL81IbdlGPbhPPG9ga7BdkLN8VbyYNCjE7kBIKM47JS1pbjeoJhI8bBrFjjp6LIJW/tWh1Pyl7Mk3DbPiZOkJ9HXvOgE/HRh44jJPtMLnzrZU5VNgNRgaEBWOG+j06pxdK9giMji1mFkJXSr43YUZYYgM3egNfNzxeTG0SshbZarRDEKeAlnDJkZ70rbP2krL2MgZJDv8vIK1PcMMFhjsJ/4Pp7F0Tl2Rm/qlZhTn4ptWagZmM0Z3N2WkNdX6Z9i2lZ5K+5jNHEFfjw/CPOFqpaFMMckpfFMsAJchbqnh+F5NvKJSFNB6L77iRCp5hbhGBbZncwc3UDO3FZ9ZuYZ8Ws+2ZyS5uVxd5ZUsvZFO+mWwySytFbsc0nUUcgkXlBiGKF/eFm9SQTURkyNzJkJfPm7awRwYoidaf8MTSp/kUCCyloAjpFIOJAa0SoVerhLp8uhQzfeU=
    root@e38-h01-000-r650.rdu2.scalelab.redhat.com
  templateRefs:
  - name: ibi-cluster-templates-v1
    namespace: open-cluster-management
status:
  clusterDeploymentRef:
    name: vm00007
  conditions:
  - lastTransitionTime: "2024-10-16T03:16:47Z"
    message: Validation succeeded
    reason: Completed
    status: "True"
    type: ClusterInstanceValidated
  - lastTransitionTime: "2024-10-16T03:16:47Z"
    message: Rendered templates successfully
    reason: Completed
    status: "True"
    type: RenderedTemplates
  - lastTransitionTime: "2024-10-16T03:18:16Z"
    message: Rendered manifests failed dry-run validation
    reason: Failed
    status: "False"
    type: RenderedTemplatesValidated
  - lastTransitionTime: "2024-10-16T03:23:07Z"
    message: Provisioning completed
    reason: Completed
    status: "True"
    type: Provisioned
  deploymentConditions:
  - lastProbeTime: "2024-10-16T13:49:18Z"
    lastTransitionTime: "2024-10-16T03:16:47Z"
    message: BareMetalHost vm00007/vm00007, is externally provisioned, skipping hardware
      validation
    reason: HostValidationSucceeded
    status: "True"
    type: ClusterInstallRequirementsMet
  - lastProbeTime: "2024-10-16T13:49:18Z"
    lastTransitionTime: "2024-10-16T03:16:47Z"
    message: Cluster installation has succeeded
    reason: ClusterInstallationSucceeded
    status: "True"
    type: ClusterInstallCompleted
  - lastProbeTime: "2024-10-16T13:49:18Z"
    lastTransitionTime: "2024-10-16T03:16:47Z"
    message: Cluster installation has succeeded
    reason: ClusterInstallationSucceeded
    status: "False"
    type: ClusterInstallFailed
  - lastProbeTime: "2024-10-16T13:49:18Z"
    lastTransitionTime: "2024-10-16T03:16:47Z"
    message: Cluster installation has succeeded
    reason: ClusterInstallationSucceeded
    status: "True"
    type: ClusterInstallStopped
  manifestsRendered:
  - apiGroup: metal3.io/v1alpha1
    kind: BareMetalHost
    lastAppliedTime: "2024-10-16T03:18:16Z"
    name: vm00007
    namespace: vm00007
    status: ""
    syncWave: 1
  - apiGroup: hive.openshift.io/v1
    kind: ClusterDeployment
    lastAppliedTime: "2024-10-16T03:18:16Z"
    name: vm00007
    namespace: vm00007
    status: ""
    syncWave: 1
  - apiGroup: extensions.hive.openshift.io/v1alpha1
    kind: ImageClusterInstall
    lastAppliedTime: "2024-10-16T03:18:16Z"
    name: vm00007
    namespace: vm00007
    status: ""
    syncWave: 1
  - apiGroup: v1
    kind: Secret
    lastAppliedTime: "2024-10-16T03:18:16Z"
    message: missing spec in rendered install template
    name: vm00007
    namespace: vm00007
    status: failed
    syncWave: 1
  - apiGroup: agent.open-cluster-management.io/v1
    kind: KlusterletAddonConfig
    lastAppliedTime: "2024-10-16T03:18:16Z"
    name: vm00007
    namespace: vm00007
    status: ""
    syncWave: 2
  - apiGroup: cluster.open-cluster-management.io/v1
    kind: ManagedCluster
    lastAppliedTime: "2024-10-16T03:18:16Z"
    name: vm00007
    status: ""
    syncWave: 2
  observedGeneration: 1

      Yaml manifest for vm00011:

      # oc get clusterinstance -n vm00011 vm00011 -o yaml
apiVersion: siteconfig.open-cluster-management.io/v1alpha1
kind: ClusterInstance
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"siteconfig.open-cluster-management.io/v1alpha1","kind":"ClusterInstance","metadata":{"annotations":{},"labels":{"app.kubernetes.io/instance":"ztp-clusters-01"},"name":"vm00011","namespace":"vm00011"},"spec":{"additionalNTPSources":["fc00:1005::21"],"baseDomain":"rdu2.scalelab.redhat.com","clusterImageSetNameRef":"openshift-4.17.1","clusterName":"vm00011","clusterNetwork":[{"cidr":"fd01::/48","hostPrefix":64}],"clusterType":"SNO","extraLabels":{"ManagedCluster":{"common":"true","du-profile":"4.17","group-du-sno":"","name":"vm00011","observability":"enabled","sites":"vm00011","vendor":"OpenShift"}},"holdInstallation":false,"installConfigOverrides":"{\"capabilities\":{\"baselineCapabilitySet\": \"None\", \"additionalEnabledCapabilities\": [ \"OperatorLifecycleManager\", \"Ingress\", \"NodeTuning\" ] }}","machineNetwork":[{"cidr":"fc00:1005::/64"}],"networkType":"OVNKubernetes","nodes":[{"automatedCleaningMode":"disabled","bmcAddress":"redfish-virtualmedia+http://[fc00:1005::21]:9000/redfish/v1/Systems/412a3810-f3d7-5078-abbe-111a20d6bbf3","bmcCredentialsName":{"name":"vm00011-bmc-secret"},"bootMACAddress":"52:54:00:00:00:0b","bootMode":"UEFI","hostName":"vm00011","ironicInspect":"disabled","nodeNetwork":{"config":{"dns-resolver":{"config":{"server":["fc00:1005::21"]}},"interfaces":[{"identifier":"mac-address","ipv4":{"enabled":false},"ipv6":{"address":[{"ip":"fc00:1005::3f2","prefix-length":64}],"autoconf":false,"dhcp":false,"enabled":true},"mac-address":"52:54:00:00:00:0b","name":"enp1s0","state":"up","type":"ethernet"}],"routes":{"config":[{"destination":"::/0","next-hop-address":"fc00:1005::1","next-hop-interface":"enp1s0"}]}},"interfaces":[{"macAddress":"52:54:00:00:00:0b","name":"enp1s0"}]},"templateRefs":[{"name":"ibi-node-templates-v1","namespace":"open-cluster-management"}]}],"pullSecretRef":{"name":"ibi-pull-secret"},"serviceNetwork":[{"cidr":"fd02::/112"}],"sshPublicKey":"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6351YHGIvE6DZcAt0RXqodzKbglUeCNqxRyd7OOfcN+p88RaKZahg9/dL81IbdlGPbhPPG9ga7BdkLN8VbyYNCjE7kBIKM47JS1pbjeoJhI8bBrFjjp6LIJW/tWh1Pyl7Mk3DbPiZOkJ9HXvOgE/HRh44jJPtMLnzrZU5VNgNRgaEBWOG+j06pxdK9giMji1mFkJXSr43YUZYYgM3egNfNzxeTG0SshbZarRDEKeAlnDJkZ70rbP2krL2MgZJDv8vIK1PcMMFhjsJ/4Pp7F0Tl2Rm/qlZhTn4ptWagZmM0Z3N2WkNdX6Z9i2lZ5K+5jNHEFfjw/CPOFqpaFMMckpfFMsAJchbqnh+F5NvKJSFNB6L77iRCp5hbhGBbZncwc3UDO3FZ9ZuYZ8Ws+2ZyS5uVxd5ZUsvZFO+mWwySytFbsc0nUUcgkXlBiGKF/eFm9SQTURkyNzJkJfPm7awRwYoidaf8MTSp/kUCCyloAjpFIOJAa0SoVerhLp8uhQzfeU= root@e38-h01-000-r650.rdu2.scalelab.redhat.com","templateRefs":[{"name":"ibi-cluster-templates-v1","namespace":"open-cluster-management"}]}}
  creationTimestamp: "2024-10-16T03:16:45Z"
  finalizers:
  - clusterinstance.siteconfig.open-cluster-management.io/finalizer
  generation: 1
  labels:
    app.kubernetes.io/instance: ztp-clusters-01
  name: vm00011
  namespace: vm00011
  resourceVersion: "1875151"
  uid: d943d391-2a76-4099-a57a-3b77162f1d0d
spec:
  additionalNTPSources:
  - fc00:1005::21
  baseDomain: rdu2.scalelab.redhat.com
  clusterImageSetNameRef: openshift-4.17.1
  clusterName: vm00011
  clusterNetwork:
  - cidr: fd01::/48
    hostPrefix: 64
  clusterType: SNO
  cpuPartitioningMode: None
  extraLabels:
    ManagedCluster:
      common: "true"
      du-profile: "4.17"
      group-du-sno: ""
      name: vm00011
      observability: enabled
      sites: vm00011
      vendor: OpenShift
  holdInstallation: false
  installConfigOverrides: '{"capabilities":{"baselineCapabilitySet": "None", "additionalEnabledCapabilities":
    [ "OperatorLifecycleManager", "Ingress", "NodeTuning" ] }}'
  machineNetwork:
  - cidr: fc00:1005::/64
  networkType: OVNKubernetes
  nodes:
  - automatedCleaningMode: disabled
    bmcAddress: redfish-virtualmedia+http://[fc00:1005::21]:9000/redfish/v1/Systems/412a3810-f3d7-5078-abbe-111a20d6bbf3
    bmcCredentialsName:
      name: vm00011-bmc-secret
    bootMACAddress: 52:54:00:00:00:0b
    bootMode: UEFI
    hostName: vm00011
    ironicInspect: disabled
    nodeNetwork:
      config:
        dns-resolver:
          config:
            server:
            - fc00:1005::21
        interfaces:
        - identifier: mac-address
          ipv4:
            enabled: false
          ipv6:
            address:
            - ip: fc00:1005::3f2
              prefix-length: 64
            autoconf: false
            dhcp: false
            enabled: true
          mac-address: 52:54:00:00:00:0b
          name: enp1s0
          state: up
          type: ethernet
        routes:
          config:
          - destination: ::/0
            next-hop-address: fc00:1005::1
            next-hop-interface: enp1s0
      interfaces:
      - macAddress: 52:54:00:00:00:0b
        name: enp1s0
    role: master
    templateRefs:
    - name: ibi-node-templates-v1
      namespace: open-cluster-management
  pullSecretRef:
    name: ibi-pull-secret
  serviceNetwork:
  - cidr: fd02::/112
  sshPublicKey: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC6351YHGIvE6DZcAt0RXqodzKbglUeCNqxRyd7OOfcN+p88RaKZahg9/dL81IbdlGPbhPPG9ga7BdkLN8VbyYNCjE7kBIKM47JS1pbjeoJhI8bBrFjjp6LIJW/tWh1Pyl7Mk3DbPiZOkJ9HXvOgE/HRh44jJPtMLnzrZU5VNgNRgaEBWOG+j06pxdK9giMji1mFkJXSr43YUZYYgM3egNfNzxeTG0SshbZarRDEKeAlnDJkZ70rbP2krL2MgZJDv8vIK1PcMMFhjsJ/4Pp7F0Tl2Rm/qlZhTn4ptWagZmM0Z3N2WkNdX6Z9i2lZ5K+5jNHEFfjw/CPOFqpaFMMckpfFMsAJchbqnh+F5NvKJSFNB6L77iRCp5hbhGBbZncwc3UDO3FZ9ZuYZ8Ws+2ZyS5uVxd5ZUsvZFO+mWwySytFbsc0nUUcgkXlBiGKF/eFm9SQTURkyNzJkJfPm7awRwYoidaf8MTSp/kUCCyloAjpFIOJAa0SoVerhLp8uhQzfeU=
    root@e38-h01-000-r650.rdu2.scalelab.redhat.com
  templateRefs:
  - name: ibi-cluster-templates-v1
    namespace: open-cluster-management
status:
  clusterDeploymentRef:
    name: vm00011
  conditions:
  - lastTransitionTime: "2024-10-16T03:17:20Z"
    message: Validation succeeded
    reason: Completed
    status: "True"
    type: ClusterInstanceValidated
  - lastTransitionTime: "2024-10-16T03:17:20Z"
    message: Rendered templates successfully
    reason: Completed
    status: "True"
    type: RenderedTemplates
  - lastTransitionTime: "2024-10-16T03:18:31Z"
    message: Rendered manifests failed dry-run validation
    reason: Failed
    status: "False"
    type: RenderedTemplatesValidated
  - lastTransitionTime: "2024-10-16T03:23:03Z"
    message: Provisioning completed
    reason: Completed
    status: "True"
    type: Provisioned
  deploymentConditions:
  - lastProbeTime: "2024-10-16T14:18:57Z"
    lastTransitionTime: "2024-10-16T03:17:20Z"
    message: BareMetalHost vm00011/vm00011, is externally provisioned, skipping hardware
      validation
    reason: HostValidationSucceeded
    status: "True"
    type: ClusterInstallRequirementsMet
  - lastProbeTime: "2024-10-16T14:18:57Z"
    lastTransitionTime: "2024-10-16T03:17:20Z"
    message: Cluster installation has succeeded
    reason: ClusterInstallationSucceeded
    status: "True"
    type: ClusterInstallCompleted
  - lastProbeTime: "2024-10-16T14:18:57Z"
    lastTransitionTime: "2024-10-16T03:17:20Z"
    message: Cluster installation has succeeded
    reason: ClusterInstallationSucceeded
    status: "False"
    type: ClusterInstallFailed
  - lastProbeTime: "2024-10-16T14:18:57Z"
    lastTransitionTime: "2024-10-16T03:17:20Z"
    message: Cluster installation has succeeded
    reason: ClusterInstallationSucceeded
    status: "True"
    type: ClusterInstallStopped
  manifestsRendered:
  - apiGroup: metal3.io/v1alpha1
    kind: BareMetalHost
    lastAppliedTime: "2024-10-16T03:18:31Z"
    name: vm00011
    namespace: vm00011
    status: ""
    syncWave: 1
  - apiGroup: hive.openshift.io/v1
    kind: ClusterDeployment
    lastAppliedTime: "2024-10-16T03:18:31Z"
    name: vm00011
    namespace: vm00011
    status: ""
    syncWave: 1
  - apiGroup: extensions.hive.openshift.io/v1alpha1
    kind: ImageClusterInstall
    lastAppliedTime: "2024-10-16T03:18:31Z"
    name: vm00011
    namespace: vm00011
    status: ""
    syncWave: 1
  - apiGroup: v1
    kind: Secret
    lastAppliedTime: "2024-10-16T03:18:31Z"
    message: missing spec in rendered install template
    name: vm00011
    namespace: vm00011
    status: failed
    syncWave: 1
  - apiGroup: agent.open-cluster-management.io/v1
    kind: KlusterletAddonConfig
    lastAppliedTime: "2024-10-16T03:18:31Z"
    name: vm00011
    namespace: vm00011
    status: ""
    syncWave: 2
  - apiGroup: cluster.open-cluster-management.io/v1
    kind: ManagedCluster
    lastAppliedTime: "2024-10-16T03:18:31Z"
    name: vm00011
    status: ""
    syncWave: 2
  observedGeneration: 1

       

        1. hub-acm-must-gather.tar.gz
          128.64 MB
          Alex Krzos
        2. must-gather-20241016-031430.tar.gz
          117.70 MB
          Alex Krzos
        3. siteconfig-controller-manager-6b8678d95-m4t85.log
          1.71 MB
          Alex Krzos

              sakhoury@redhat.com Sharat Akhoury
              akrzos@redhat.com Alex Krzos
              Votes:
              0 Vote for this issue
              Watchers:
              1 Start watching this issue

                Created:
                Updated: