Uploaded image for project: 'OpenShift Logging'
  1. OpenShift Logging
  2. LOG-2108

Elasticsearch operator status unknown and reporting error previously used GenUUID is no longer found in Spec.Nodes

    XMLWordPrintable

Details

    • Bug
    • Resolution: Done
    • Undefined
    • None
    • Logging 5.4.0
    • Log Storage
    • None
    • False
    • False
    • NEW
    • NEW
    • Logging (LogExp) - Sprint 214

    Description

      Elasticsearch operator status is unknown and reporting error 'previously used GenUUID is no longer found in Spec.Nodes'

      Server Version: 4.10.0-0.nightly-2021-12-23-153012
      Kubernetes Version: v1.22.1+6859754

      Steps to reproduce the issue:

      1 Deploy Logging 5.4

      oc get csv
      NAME                              DISPLAY                            VERSION    REPLACES   PHASE
      cluster-logging.5.4.0-11          Red Hat OpenShift Logging          5.4.0-11              Succeeded
      elasticsearch-operator.5.4.0-25   OpenShift Elasticsearch Operator   5.4.0-25              Succeeded 

      2 Create a CLO instance.

      apiVersion: "logging.openshift.io/v1"
      kind: "ClusterLogging"
      metadata:
        name: "instance" 
        namespace: "openshift-logging"
      spec:
        managementState: "Managed"  
        logStore:
          type: "elasticsearch"  
          retentionPolicy: 
            application:
              maxAge: 7d
            infra:
              maxAge: 7d
            audit:
              maxAge: 7d
          elasticsearch:
            nodeCount: 3 
            storage: {} 
            resources: 
                limits:
                  memory: "4Gi"
                requests:
                  memory: "1Gi"
            proxy: 
              resources:
                limits:
                  memory: 256Mi
                requests:
                  memory: 256Mi
            redundancyPolicy: "SingleRedundancy"
        visualization:
          type: "kibana"  
          kibana:
            replicas: 1
        collection:
          logs:
            type: "fluentd"  
            fluentd: {} 

      3 Check the Elasticsearch and Cluster Logging instance status.

      oc get clusterloggings.logging.openshift.io instance -o yaml
      apiVersion: logging.openshift.io/v1
      kind: ClusterLogging
      metadata:
        creationTimestamp: "2021-12-31T06:02:38Z"
        generation: 1
        name: instance
        namespace: openshift-logging
        resourceVersion: "90749"
        uid: 1c2e5d6a-cf93-4b6a-843b-29caf6f26666
      spec:
        collection:
          logs:
            fluentd: {}
            type: fluentd
        logStore:
          elasticsearch:
            nodeCount: 3
            proxy:
              resources:
                limits:
                  memory: 256Mi
                requests:
                  memory: 256Mi
            redundancyPolicy: SingleRedundancy
            resources:
              limits:
                memory: 4Gi
              requests:
                memory: 1Gi
            storage: {}
          retentionPolicy:
            application:
              maxAge: 7d
            audit:
              maxAge: 7d
            infra:
              maxAge: 7d
          type: elasticsearch
        managementState: Managed
        visualization:
          kibana:
            replicas: 1
          type: kibana
      status:
        clusterConditions:
        - lastTransitionTime: "2021-12-31T06:02:51Z"
          status: "False"
          type: CollectorDeadEnd
        - lastTransitionTime: "2021-12-31T06:02:38Z"
          message: curator is deprecated in favor of defining retention policy
          reason: ResourceDeprecated
          status: "True"
          type: CuratorRemoved
        collection:
          logs:
            fluentdStatus:
              daemonSet: collector
              nodes:
                collector-25xx4: ip-10-0-218-19.us-east-2.compute.internal
                collector-cq9nv: ip-10-0-172-196.us-east-2.compute.internal
                collector-hj8q9: ip-10-0-133-30.us-east-2.compute.internal
                collector-q6674: ip-10-0-149-128.us-east-2.compute.internal
                collector-s8576: ip-10-0-176-112.us-east-2.compute.internal
                collector-wt6fm: ip-10-0-204-125.us-east-2.compute.internal
              pods:
                failed: []
                notReady: []
                ready:
                - collector-25xx4
                - collector-cq9nv
                - collector-hj8q9
                - collector-q6674
                - collector-s8576
                - collector-wt6fm
        curation: {}
        logStore:
          elasticsearchStatus:
          - cluster:
              activePrimaryShards: 0
              activeShards: 0
              initializingShards: 0
              numDataNodes: 0
              numNodes: 0
              pendingTasks: 0
              relocatingShards: 0
              status: cluster health unknown
              unassignedShards: 0
            clusterConditions:
            - lastTransitionTime: "2021-12-31T06:02:42Z"
              message: previously used GenUUID is no longer found in Spec.Nodes
              reason: Invalid Spec
              status: "True"
              type: InvalidUUID
            clusterName: elasticsearch
            nodeConditions:
              elasticsearch-cdm-45xnj4t6-1:
              - lastTransitionTime: "2021-12-31T06:02:42Z"
                reason: ContainerCreating
                status: "True"
                type: ElasticsearchContainerWaiting
              - lastTransitionTime: "2021-12-31T06:02:42Z"
                reason: ContainerCreating
                status: "True"
                type: ProxyContainerWaiting
              elasticsearch-cdm-45xnj4t6-2:
              - lastTransitionTime: "2021-12-31T06:02:42Z"
                reason: ContainerCreating
                status: "True"
                type: ElasticsearchContainerWaiting
              - lastTransitionTime: "2021-12-31T06:02:42Z"
                reason: ContainerCreating
                status: "True"
                type: ProxyContainerWaiting
              elasticsearch-cdm-45xnj4t6-3:
              - lastTransitionTime: "2021-12-31T06:02:42Z"
                reason: ContainerCreating
                status: "True"
                type: ElasticsearchContainerWaiting
              - lastTransitionTime: "2021-12-31T06:02:42Z"
                reason: ContainerCreating
                status: "True"
                type: ProxyContainerWaiting
            nodeCount: 0
            pods:
              client:
                failed: []
                notReady:
                - elasticsearch-cdm-45xnj4t6-1-569cd6cbd4-nghdl
                - elasticsearch-cdm-45xnj4t6-2-5f7f875d7d-h22wn
                - elasticsearch-cdm-45xnj4t6-3-5f689d98d-bwwcx
                ready: []
              data:
                failed: []
                notReady:
                - elasticsearch-cdm-45xnj4t6-1-569cd6cbd4-nghdl
                - elasticsearch-cdm-45xnj4t6-2-5f7f875d7d-h22wn
                - elasticsearch-cdm-45xnj4t6-3-5f689d98d-bwwcx
                ready: []
              master:
                failed: []
                notReady:
                - elasticsearch-cdm-45xnj4t6-1-569cd6cbd4-nghdl
                - elasticsearch-cdm-45xnj4t6-2-5f7f875d7d-h22wn
                - elasticsearch-cdm-45xnj4t6-3-5f689d98d-bwwcx
                ready: []
            shardAllocationEnabled: shard allocation unknown
        visualization:
          kibanaStatus:
          - deployment: kibana
            pods:
              failed: []
              notReady: []
              ready:
              - kibana-55c9bb6984-hk2pc
            replicaSets:
            - kibana-55c9bb6984
            replicas: 1
      
      oc get elasticsearches.logging.openshift.io elasticsearch -o yaml 
      apiVersion: logging.openshift.io/v1
      kind: Elasticsearch
      metadata:
        creationTimestamp: "2021-12-31T06:02:38Z"
        generation: 1
        name: elasticsearch
        namespace: openshift-logging
        ownerReferences:
        - apiVersion: logging.openshift.io/v1
          controller: true
          kind: ClusterLogging
          name: instance
          uid: 1c2e5d6a-cf93-4b6a-843b-29caf6f26666
        resourceVersion: "89997"
        uid: 7e1fb36b-21b9-47c2-92ed-565fdb5bb24a
      spec:
        indexManagement:
          mappings:
          - aliases:
            - app
            - logs.app
            name: app
            policyRef: app-policy
          - aliases:
            - infra
            - logs.infra
            name: infra
            policyRef: infra-policy
          - aliases:
            - audit
            - logs.audit
            name: audit
            policyRef: audit-policy
          policies:
          - name: app-policy
            phases:
              delete:
                minAge: 7d
              hot:
                actions:
                  rollover:
                    maxAge: 8h
            pollInterval: 15m
          - name: infra-policy
            phases:
              delete:
                minAge: 7d
              hot:
                actions:
                  rollover:
                    maxAge: 8h
            pollInterval: 15m
          - name: audit-policy
            phases:
              delete:
                minAge: 7d
              hot:
                actions:
                  rollover:
                    maxAge: 8h
            pollInterval: 15m
        managementState: Managed
        nodeSpec:
          proxyResources:
            limits:
              memory: 256Mi
            requests:
              memory: 256Mi
          resources:
            limits:
              memory: 4Gi
            requests:
              memory: 1Gi
        nodes:
        - nodeCount: 3
          proxyResources: {}
          resources: {}
          roles:
          - client
          - data
          - master
          storage: {}
        redundancyPolicy: SingleRedundancy
      status:
        cluster:
          activePrimaryShards: 0
          activeShards: 0
          initializingShards: 0
          numDataNodes: 0
          numNodes: 0
          pendingTasks: 0
          relocatingShards: 0
          status: cluster health unknown
          unassignedShards: 0
        conditions:
        - lastTransitionTime: "2021-12-31T06:02:42Z"
          message: previously used GenUUID is no longer found in Spec.Nodes
          reason: Invalid Spec
          status: "True"
          type: InvalidUUID
        nodes:
        - conditions:
          - lastTransitionTime: "2021-12-31T06:02:42Z"
            reason: ContainerCreating
            status: "True"
            type: ElasticsearchContainerWaiting
          - lastTransitionTime: "2021-12-31T06:02:42Z"
            reason: ContainerCreating
            status: "True"
            type: ProxyContainerWaiting
          deploymentName: elasticsearch-cdm-45xnj4t6-1
          upgradeStatus: {}
        - conditions:
          - lastTransitionTime: "2021-12-31T06:02:42Z"
            reason: ContainerCreating
            status: "True"
            type: ElasticsearchContainerWaiting
          - lastTransitionTime: "2021-12-31T06:02:42Z"
            reason: ContainerCreating
            status: "True"
            type: ProxyContainerWaiting
          deploymentName: elasticsearch-cdm-45xnj4t6-2
          upgradeStatus: {}
        - conditions:
          - lastTransitionTime: "2021-12-31T06:02:42Z"
            reason: ContainerCreating
            status: "True"
            type: ElasticsearchContainerWaiting
          - lastTransitionTime: "2021-12-31T06:02:42Z"
            reason: ContainerCreating
            status: "True"
            type: ProxyContainerWaiting
          deploymentName: elasticsearch-cdm-45xnj4t6-3
          upgradeStatus: {}
        pods:
          client:
            failed: []
            notReady:
            - elasticsearch-cdm-45xnj4t6-1-569cd6cbd4-nghdl
            - elasticsearch-cdm-45xnj4t6-2-5f7f875d7d-h22wn
            - elasticsearch-cdm-45xnj4t6-3-5f689d98d-bwwcx
            ready: []
          data:
            failed: []
            notReady:
            - elasticsearch-cdm-45xnj4t6-1-569cd6cbd4-nghdl
            - elasticsearch-cdm-45xnj4t6-2-5f7f875d7d-h22wn
            - elasticsearch-cdm-45xnj4t6-3-5f689d98d-bwwcx
            ready: []
          master:
            failed: []
            notReady:
            - elasticsearch-cdm-45xnj4t6-1-569cd6cbd4-nghdl
            - elasticsearch-cdm-45xnj4t6-2-5f7f875d7d-h22wn
            - elasticsearch-cdm-45xnj4t6-3-5f689d98d-bwwcx
            ready: []
        shardAllocationEnabled: shard allocation unknown
       

      Attachments

        Activity

          People

            btaani@redhat.com Bayan Taani
            rhn-support-ikanse Ishwar Kanse
            Ishwar Kanse Ishwar Kanse
            Votes:
            0 Vote for this issue
            Watchers:
            3 Start watching this issue

            Dates

              Created:
              Updated:
              Resolved: