hypershift-mce-install step occasionally fails

 master    

    30%

    1. mce-install step
    2.
    3.
    

    + '[' -f /tmp/secret/proxy-conf.sh ']'
+ source /tmp/secret/proxy-conf.sh
++ export PROXYPORT=8213
++ PROXYPORT=8213
++ export HTTP_PROXY=http://145.40.99.35:8213/
++ HTTP_PROXY=http://145.40.99.35:8213/
++ export HTTPS_PROXY=http://145.40.99.35:8213/
++ HTTPS_PROXY=http://145.40.99.35:8213/
++ export NO_PROXY=static.redhat.com,redhat.io,quay.io,openshift.org,openshift.com,svc,amazonaws.com,r2.cloudflarestorage.com,github.com,githubusercontent.com,google.com,googleapis.com,fedoraproject.org,cloudfront.net,localhost,127.0.0.1
++ NO_PROXY=static.redhat.com,redhat.io,quay.io,openshift.org,openshift.com,svc,amazonaws.com,r2.cloudflarestorage.com,github.com,githubusercontent.com,google.com,googleapis.com,fedoraproject.org,cloudfront.net,localhost,127.0.0.1
++ export http_proxy=http://145.40.99.35:8213/
++ http_proxy=http://145.40.99.35:8213/
++ export https_proxy=http://145.40.99.35:8213/
++ https_proxy=http://145.40.99.35:8213/
++ export no_proxy=static.redhat.com,redhat.io,quay.io,openshift.org,openshift.com,svc,amazonaws.com,r2.cloudflarestorage.com,github.com,githubusercontent.com,google.com,googleapis.com,fedoraproject.org,cloudfront.net,localhost,127.0.0.1
++ no_proxy=static.redhat.com,redhat.io,quay.io,openshift.org,openshift.com,svc,amazonaws.com,r2.cloudflarestorage.com,github.com,githubusercontent.com,google.com,googleapis.com,fedoraproject.org,cloudfront.net,localhost,127.0.0.1
+ MCE_VERSION=2.6
+ [[ false != \t\r\u\e ]]
+ _REPO=quay.io/acm-d/mce-custom-registry
+ cat
+ oc apply -f -
imagecontentsourcepolicy.operator.openshift.io/rhacm-repo created
++ cat /etc/acm-d-mce-quay-pull-credentials/acm_d_mce_quay_username
+ QUAY_USERNAME=XXXXXXXXXXXXXXXXX
++ cat /etc/acm-d-mce-quay-pull-credentials/acm_d_mce_quay_pullsecret
+ QUAY_PASSWORD=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+ oc get secret pull-secret -n openshift-config -o json
+ jq -r '.data.".dockerconfigjson"'
+ base64 -d
++ echo -n XXXXXXXXXXXXXXXXX:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
++ base64 -w 0
+ QUAY_AUTH=YWNtLWQrbGlhbmdsaXB1bGw6XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
+ jq --arg QUAY_AUTH YWNtLWQrbGlhbmdsaXB1bGw6XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX '.auths += {"quay.io:443": {"auth":$QUAY_AUTH,"email":""}}' /tmp/global-pull-secret.json
+ mv /tmp/global-pull-secret.json.tmp /tmp/global-pull-secret.json
+ oc set data secret/pull-secret -n openshift-config --from-file=.dockerconfigjson=/tmp/global-pull-secret.json
secret/pull-secret data updated
+ rm /tmp/global-pull-secret.json
+ sleep 60
+ oc wait mcp master worker --for condition=updated --timeout=20m
machineconfigpool.machineconfiguration.openshift.io/master condition met
machineconfigpool.machineconfiguration.openshift.io/worker condition met
++ oc version
++ grep 'Client Version:'
+ VER='Client Version: v4.2.0-alpha.0-2352-gcf533b5'
+ echo '* oc CLI Client Version: v4.2.0-alpha.0-2352-gcf533b5'
* oc CLI Client Version: v4.2.0-alpha.0-2352-gcf533b5
+ echo 'Install MCE custom catalog source'
Install MCE custom catalog source
+ IMG=quay.io/acm-d/mce-custom-registry:2.6-latest
+ oc apply -f -
catalogsource.operators.coreos.com/multiclusterengine-catalog created
+ oc apply -f -
namespace/multicluster-engine created
+ oc apply -f -
operatorgroup.operators.coreos.com/multicluster-engine-group created
++ [[ false == \t\r\u\e ]]
++ echo -n multiclusterengine-catalog
+ CATALOG=multiclusterengine-catalog
+ echo '* Applying SUBSCRIPTION_CHANNEL 2.6, SUBSCRIPTION_SOURCE multiclusterengine-catalog to multiclusterengine-operator subscription'
* Applying SUBSCRIPTION_CHANNEL 2.6, SUBSCRIPTION_SOURCE multiclusterengine-catalog to multiclusterengine-operator subscription
+ oc apply -f -
subscription.operators.coreos.com/multicluster-engine created
+ CSVName=
+ (( i=1 ))
+ (( i<=60 ))
++ oc get sub multicluster-engine -n multicluster-engine -o 'jsonpath={.status.currentCSV}'
++ echo exists
+ output=exists
+ '[' exists '!=' exists ']'
++ oc get sub -n multicluster-engine multicluster-engine -o 'jsonpath={.status.currentCSV}'
+ CSVName=
+ '[' '' '!=' '' ']'
+ sleep 10
+ (( i++ ))
+ (( i<=60 ))
++ oc get sub multicluster-engine -n multicluster-engine -o 'jsonpath={.status.currentCSV}'
++ echo exists
+ output=exists
+ '[' exists '!=' exists ']'
++ oc get sub -n multicluster-engine multicluster-engine -o 'jsonpath={.status.currentCSV}'
+ CSVName=
+ '[' '' '!=' '' ']'
+ sleep 10
+ (( i++ ))
+ (( i<=60 ))
++ oc get sub multicluster-engine -n multicluster-engine -o 'jsonpath={.status.currentCSV}'
++ echo exists
+ output=exists
+ '[' exists '!=' exists ']'
++ oc get sub -n multicluster-engine multicluster-engine -o 'jsonpath={.status.currentCSV}'
+ CSVName=
+ '[' '' '!=' '' ']'
+ sleep 10
+ (( i++ ))
+ (( i<=60 ))
++ oc get sub multicluster-engine -n multicluster-engine -o 'jsonpath={.status.currentCSV}'
++ echo exists
+ output=exists
+ '[' exists '!=' exists ']'
++ oc get sub -n multicluster-engine multicluster-engine -o 'jsonpath={.status.currentCSV}'
+ CSVName=
+ '[' '' '!=' '' ']'
+ sleep 10
+ (( i++ ))
+ (( i<=60 ))
++ oc get sub multicluster-engine -n multicluster-engine -o 'jsonpath={.status.currentCSV}'
++ echo exists
+ output=exists
+ '[' exists '!=' exists ']'
++ oc get sub -n multicluster-engine multicluster-engine -o 'jsonpath={.status.currentCSV}'
+ CSVName=multicluster-engine.v2.6.4
+ '[' multicluster-engine.v2.6.4 '!=' '' ']'
+ break
+ _apiReady=0
+ echo '* Using CSV: multicluster-engine.v2.6.4'
* Using CSV: multicluster-engine.v2.6.4
+ (( i=1 ))
+ (( i<=20 ))
+ sleep 30
++ oc get csv -n multicluster-engine multicluster-engine.v2.6.4 -o 'jsonpath={.status.phase}'
++ echo exists
+ output=exists
+ '[' exists '!=' exists ']'
++ oc get csv -n multicluster-engine multicluster-engine.v2.6.4 -o 'jsonpath={.status.phase}'
+ phase=Succeeded
+ '[' Succeeded == Succeeded ']'
+ _apiReady=1
+ break
+ '[' 1 -eq 0 ']'
+ echo 'multiclusterengine installed successfully'
multiclusterengine installed successfully
+ oc apply -f -
multiclusterengine.multicluster.openshift.io/multiclusterengine-sample created
+ sleep 5
+ oc patch mce multiclusterengine-sample --type=merge -p '{"spec":{"overrides":{"components":[{"name":"hypershift-preview","enabled": true}]}}}'
multiclusterengine.multicluster.openshift.io/multiclusterengine-sample patched
+ echo 'wait for mce to Available'
wait for mce to Available
+ oc wait --timeout=20m --for=condition=Available MultiClusterEngine/multiclusterengine-sample
multiclusterengine.multicluster.openshift.io/multiclusterengine-sample condition met
+ oc apply -f -
Warning: resource managedclusters/local-cluster is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by oc apply. oc apply should only be used on resources created declaratively by either oc create --save-config or oc apply. The missing annotation will be patched automatically.
managedcluster.cluster.open-cluster-management.io/local-cluster configured
+ oc wait --timeout=5m --for=condition=HubAcceptedManagedCluster -n local-cluster ManagedCluster/local-cluster
managedcluster.cluster.open-cluster-management.io/local-cluster condition met
+ oc wait --timeout=5m --for=condition=ManagedClusterImportSucceeded -n local-cluster ManagedCluster/local-cluster
managedcluster.cluster.open-cluster-management.io/local-cluster condition met
+ oc wait --timeout=5m --for=condition=ManagedClusterConditionAvailable -n local-cluster ManagedCluster/local-cluster
managedcluster.cluster.open-cluster-management.io/local-cluster condition met
+ oc wait --timeout=5m --for=condition=ManagedClusterJoined -n local-cluster ManagedCluster/local-cluster
managedcluster.cluster.open-cluster-management.io/local-cluster condition met
+ echo 'MCE local-cluster is ready!'
MCE local-cluster is ready!
+ oc apply -f -
Warning: resource managedclusteraddons/hypershift-addon is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by oc apply. oc apply should only be used on resources created declaratively by either oc create --save-config or oc apply. The missing annotation will be patched automatically.
managedclusteraddon.addon.open-cluster-management.io/hypershift-addon configured
+ _hypershiftReady=0
+ set +e
+ (( i=1 ))
+ (( i<=20 ))
+ oc get pods -n hypershift
+ grep 'operator.*Running'
operator-5785df899c-x9ctn   1/1     Running       0          65s
operator-698db657df-xg4bl   1/1     Running       0          62s
+ '[' 0 -eq 0 ']'
+ _hypershiftReady=1
+ break
+ set -e
+ '[' 1 -eq 0 ']'
+ echo 'hypershift is running! Waiting for the pods to become ready'
hypershift is running! Waiting for the pods to become ready
+ oc wait --timeout=5m --for=condition=Ready -n hypershift pod -l name=operator
pod/operator-5785df899c-x9ctn condition met
pod/operator-698db657df-q569j condition met
pod/operator-698db657df-xg4bl condition met
Error from server (NotFound): pods "operator-5785df899c-97ssh" not found

    can be successful