STEP-TEST-ARGO-ROLLOUTS + RESULT=/install-dir/psi-417-result.txt Logging in as kubeadmin user + trap update_result EXIT + echo -e 'Logging in as kubeadmin user' ++ cat /install-dir/psi-417/auth/kubeadmin-password ++ cat /install-dir/psi-417/auth/api-url + oc login -u kubeadmin -p nw5CS-XEvh8-3jZpw-SgwvS https://api.psi-417.ocp-gitops-qe.com:6443 --insecure-skip-tls-verify=true WARNING: Using insecure TLS client config. Setting this option is not supported! Login successful. You have access to 78 projects, the list has been suppressed. You can list all projects with 'oc projects' Using project "default". Welcome! See 'oc help' to get started. ++ echo true ++ tr '[:upper:]' '[:lower:]' + [[ true == \f\a\l\s\e ]] + exit_code=0 + failed=0 ++ oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o 'jsonpath={.metadata.namespace}' --ignore-not-found + openshift_gitops_operator= + operator_namespace=openshift-operators + oc patch subscription openshift-gitops-operator -n openshift-operators --type merge --patch '{"spec": {"config": {"env": [{"name": "CLUSTER_SCOPED_ARGO_ROLLOUTS_NAMESPACES", "value": "argo-rollouts,test-rom-ns-1,rom-ns-1"}]}}}' subscription.operators.coreos.com/openshift-gitops-operator patched ++ mktemp -d + ROLLOUTS_TMP_DIR=/tmp/tmp.TxJP1GNlai + cd /tmp/tmp.TxJP1GNlai + git clone https://github.com/argoproj-labs/argo-rollouts-manager Cloning into 'argo-rollouts-manager'... + cd /tmp/tmp.TxJP1GNlai/argo-rollouts-manager + TARGET_ROLLOUT_MANAGER_COMMIT=e054d35fd0d9dad6b9ae26efa60f43cadc194b4b + TARGET_OPENSHIFT_ROUTE_ROLLOUT_PLUGIN_COMMIT=8b4125a7f9ecffb0247df91a4c890f88c0c523b7 + git checkout e054d35fd0d9dad6b9ae26efa60f43cadc194b4b Note: switching to 'e054d35fd0d9dad6b9ae26efa60f43cadc194b4b'. You are in 'detached HEAD' state. You can look around, make experimental changes and commit them, and you can discard any commits you make in this state without impacting any branches by switching back to a branch. If you want to create a new branch to retain commits you create, you may do so (now or later) by using -c with the switch command. Example: git switch -c Or undo this operation with: git switch - Turn off this advice by setting config variable advice.detachedHead to false HEAD is now at e054d35 Backport cluster/namespace-scoped RBAC tests to 0.0.3 + make test-e2e NAMESPACE_SCOPED_ARGO_ROLLOUTS=false hack/run-rollouts-manager-e2e-tests.sh + '[' false == true ']' + go test -v -p=1 -timeout=30m -race -count=1 -coverprofile=coverage.out ./tests/e2e/cluster-scoped go: downloading sigs.k8s.io/controller-runtime v0.16.3 go: downloading github.com/onsi/gomega v1.27.10 go: downloading k8s.io/client-go v0.28.3 go: downloading k8s.io/api v0.28.3 go: downloading k8s.io/apimachinery v0.28.3 go: downloading github.com/onsi/ginkgo/v2 v2.11.0 go: downloading go.uber.org/zap v1.25.0 go: downloading github.com/coreos/prometheus-operator v0.40.0 go: downloading github.com/go-logr/logr v1.2.4 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading k8s.io/apiextensions-apiserver v0.28.3 go: downloading sigs.k8s.io/yaml v1.3.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/go-logr/zapr v1.2.4 go: downloading github.com/evanphx/json-patch/v5 v5.6.0 go: downloading github.com/evanphx/json-patch v5.6.0+incompatible go: downloading k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/klog/v2 v2.100.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.2.3 go: downloading golang.org/x/net v0.23.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 go: downloading github.com/pkg/errors v0.9.1 go: downloading golang.org/x/sys v0.18.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading github.com/json-iterator/go v1.1.12 go: downloading gomodules.xyz/jsonpatch/v2 v2.4.0 go: downloading k8s.io/component-base v0.28.3 go: downloading golang.org/x/text v0.14.0 go: downloading golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 go: downloading github.com/prometheus/client_golang v1.16.0 go: downloading github.com/google/gnostic-models v0.6.8 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/golang/protobuf v1.5.3 go: downloading github.com/imdario/mergo v0.3.13 go: downloading github.com/spf13/pflag v1.0.5 go: downloading golang.org/x/term v0.18.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading golang.org/x/time v0.3.0 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/fsnotify/fsnotify v1.6.0 go: downloading golang.org/x/oauth2 v0.10.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 go: downloading github.com/go-openapi/jsonreference v0.20.2 go: downloading github.com/go-openapi/swag v0.22.3 go: downloading github.com/prometheus/common v0.44.0 go: downloading github.com/prometheus/client_model v0.4.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.2.0 go: downloading github.com/prometheus/procfs v0.10.1 go: downloading github.com/emicklei/go-restful/v3 v3.11.0 go: downloading github.com/go-openapi/jsonpointer v0.19.6 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.4 go: downloading github.com/josharian/intern v1.0.0 === RUN TestClusterScoped Running Suite: Cluster-scoped Suite - /tmp/tmp.TxJP1GNlai/argo-rollouts-manager/tests/e2e/cluster-scoped ======================================================================================================== Random Seed: 1732705063 Will run 6 of 6 specs HavePhase: expected: Available actual: HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Available HaveSuccessCondition: expected: {Reconciled True 0 0001-01-01 00:00:00 +0000 UTC Success } actual: {Reconciled True 0 2024-11-27 10:58:08 +0000 UTC Success } Object exists in ExistByName: argo-rollouts Object exists in ExistByName: argo-rollouts ------------------------------ • [FAILED] [40.969 seconds] Cluster-scoped RolloutManager tests Testing cluster-scoped RolloutManager behaviour [It] After creating cluster-scoped RolloutManager in one namespace, operator should create appropriate K8s resources and watch argo rollouts CR in another namespace. /tmp/tmp.TxJP1GNlai/argo-rollouts-manager/tests/e2e/cluster-scoped/cluster_scoped_rollouts_test.go:49 Timeline >> STEP: Create cluster-scoped RolloutManager in default namespace. @ 11/27/24 10:58:08.752 STEP: Verify that RolloutManager is successfully created. @ 11/27/24 10:58:08.783 STEP: Verify that Status.Condition is having success condition. @ 11/27/24 10:58:24.273 STEP: Verify that expected resources are created. @ 11/27/24 10:58:24.302 STEP: Verify that ServiceAccount is created. @ 11/27/24 10:58:24.302 STEP: Verify that ServiceAccount has correct labels and annotations. @ 11/27/24 10:58:24.329 STEP: Verify that argo-rollout ClusterRoles is created. @ 11/27/24 10:58:24.329 STEP: Verify that ClusterRole has correct labels and annotations. @ 11/27/24 10:58:24.354 STEP: Verify that ClusterRole has correct policy rules. @ 11/27/24 10:58:24.354 [FAILED] in [It] - /tmp/tmp.TxJP1GNlai/argo-rollouts-manager/tests/e2e/test_utils.go:215 @ 11/27/24 10:58:24.36 << Timeline [FAILED] Expected <[]v1.PolicyRule | len:23, cap:34>: [ { Verbs: ["get", "list", "watch", "update", "patch"], APIGroups: ["argoproj.io"], Resources: [ "rollouts", "rollouts/status", "rollouts/finalizers", ], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "list", "watch", "update", "patch", "delete"], APIGroups: ["argoproj.io"], Resources: [ "analysisruns", "analysisruns/finalizers", "experiments", "experiments/finalizers", ], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["get", "list", "watch"], APIGroups: ["argoproj.io"], Resources: [ "analysistemplates", "clusteranalysistemplates", ], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "list", "watch", "update", "patch", "delete"], APIGroups: ["apps"], Resources: ["replicasets"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["get", "list", "watch", "update", "patch"], APIGroups: ["", "apps"], Resources: ["deployments", "podtemplates"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["get", "list", "watch", "patch", "create", "delete"], APIGroups: [""], Resources: ["services"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "update"], APIGroups: [ "coordination.k8s.io", ], Resources: ["leases"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["get", "list", "watch"], APIGroups: [""], Resources: ["secrets", "configmaps"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["list", "update", "watch"], APIGroups: [""], Resources: ["pods"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create"], APIGroups: [""], Resources: ["pods/eviction"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "update", "patch"], APIGroups: [""], Resources: ["events"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "list", "watch", "patch"], APIGroups: ["networking.k8s.io", "extensions"], Resources: ["ingresses"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "list", "watch", "update", "patch", "delete"], APIGroups: ["batch"], Resources: ["jobs"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["watch", "get", "update", "patch", "list"], APIGroups: [ "networking.istio.io", ], Resources: ["virtualservices", "destinationrules"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "watch", "get", "update", "patch"], APIGroups: ["split.smi-spec.io"], Resources: ["trafficsplits"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "watch", "get", "update", "list", "delete"], APIGroups: ["getambassador.io", "x.getambassador.io"], Resources: ["mappings", "... Gomega truncated this representation as it exceeds 'format.MaxLength'. Consider having the object provide a custom 'GomegaStringer' representation or adjust the parameters in Gomega's 'format' package. Learn more here: https://onsi.github.io/gomega/#adjusting-output to consist of <[]v1.PolicyRule | len:23, cap:23>: [ { Verbs: ["get", "list", "watch", "update", "patch"], APIGroups: ["argoproj.io"], Resources: [ "rollouts", "rollouts/status", "rollouts/finalizers", ], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "list", "watch", "update", "patch", "delete"], APIGroups: ["argoproj.io"], Resources: [ "analysisruns", "analysisruns/finalizers", "experiments", "experiments/finalizers", ], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["get", "list", "watch"], APIGroups: ["argoproj.io"], Resources: [ "analysistemplates", "clusteranalysistemplates", ], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "list", "watch", "update", "patch", "delete"], APIGroups: ["apps"], Resources: ["replicasets"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["get", "list", "watch"], APIGroups: ["", "apps"], Resources: ["deployments", "podtemplates"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["get", "list", "watch", "patch", "create", "delete"], APIGroups: [""], Resources: ["services"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "update"], APIGroups: [ "coordination.k8s.io", ], Resources: ["leases"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["get", "list", "watch"], APIGroups: [""], Resources: ["secrets", "configmaps"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["list", "update", "watch"], APIGroups: [""], Resources: ["pods"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create"], APIGroups: [""], Resources: ["pods/eviction"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "update", "patch"], APIGroups: [""], Resources: ["events"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "list", "watch", "patch"], APIGroups: ["networking.k8s.io", "extensions"], Resources: ["ingresses"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "list", "watch", "update", "patch", "delete"], APIGroups: ["batch"], Resources: ["jobs"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["watch", "get", "update", "patch", "list"], APIGroups: [ "networking.istio.io", ], Resources: ["virtualservices", "destinationrules"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "watch", "get", "update", "patch"], APIGroups: ["split.smi-spec.io"], Resources: ["trafficsplits"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "watch", "get", "update", "list", "delete"], APIGroups: ["getambassador.io", "x.getambassador.io"], Resources: ["mappings", "ambassadormappings"... Gomega truncated this representation as it exceeds 'format.MaxLength'. Consider having the object provide a custom 'GomegaStringer' representation or adjust the parameters in Gomega's 'format' package. Learn more here: https://onsi.github.io/gomega/#adjusting-output the missing elements were <[]v1.PolicyRule | len:2, cap:2>: [ { Verbs: ["get", "list", "watch"], APIGroups: ["", "apps"], Resources: ["deployments", "podtemplates"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["watch", "get", "update"], APIGroups: [ "traefik.containo.us", ], Resources: ["traefikservices"], ResourceNames: nil, NonResourceURLs: nil, }, ] the extra elements were <[]v1.PolicyRule | len:2, cap:2>: [ { Verbs: ["get", "list", "watch", "update", "patch"], APIGroups: ["", "apps"], Resources: ["deployments", "podtemplates"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["watch", "get", "update"], APIGroups: [ "traefik.containo.us", "traefik.io", ], Resources: ["traefikservices"], ResourceNames: nil, NonResourceURLs: nil, }, ] In [It] at: /tmp/tmp.TxJP1GNlai/argo-rollouts-manager/tests/e2e/test_utils.go:215 @ 11/27/24 10:58:24.36 ------------------------------ HavePhase: expected: Available actual: HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Available HaveSuccessCondition: expected: {Reconciled True 0 0001-01-01 00:00:00 +0000 UTC Success } actual: {Reconciled True 0 2024-11-27 10:58:39 +0000 UTC Success } Object exists in ExistByName: argo-rollouts Object exists in ExistByName: argo-rollouts • [FAILED] [26.765 seconds] Cluster-scoped RolloutManager tests Testing cluster-scoped RolloutManager behaviour [It] After creating cluster-scoped RolloutManager in a namespace, operator should create appropriate K8s resources and watch argo rollouts CR in multiple namespace. /tmp/tmp.TxJP1GNlai/argo-rollouts-manager/tests/e2e/cluster-scoped/cluster_scoped_rollouts_test.go:90 Timeline >> STEP: Create a namespace for RolloutManager. @ 11/27/24 10:58:39.623 STEP: Create cluster-scoped RolloutManager. @ 11/27/24 10:58:39.653 STEP: Verify that RolloutManager is successfully created. @ 11/27/24 10:58:39.682 STEP: Verify that Status.Condition is having success condition. @ 11/27/24 10:58:51.054 STEP: Verify that expected resources are created. @ 11/27/24 10:58:51.083 STEP: Verify that ServiceAccount is created. @ 11/27/24 10:58:51.083 STEP: Verify that ServiceAccount has correct labels and annotations. @ 11/27/24 10:58:51.096 STEP: Verify that argo-rollout ClusterRoles is created. @ 11/27/24 10:58:51.096 STEP: Verify that ClusterRole has correct labels and annotations. @ 11/27/24 10:58:51.12 STEP: Verify that ClusterRole has correct policy rules. @ 11/27/24 10:58:51.12 [FAILED] in [It] - /tmp/tmp.TxJP1GNlai/argo-rollouts-manager/tests/e2e/test_utils.go:215 @ 11/27/24 10:58:51.126 << Timeline [FAILED] Expected <[]v1.PolicyRule | len:23, cap:34>: [ { Verbs: ["get", "list", "watch", "update", "patch"], APIGroups: ["argoproj.io"], Resources: [ "rollouts", "rollouts/status", "rollouts/finalizers", ], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "list", "watch", "update", "patch", "delete"], APIGroups: ["argoproj.io"], Resources: [ "analysisruns", "analysisruns/finalizers", "experiments", "experiments/finalizers", ], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["get", "list", "watch"], APIGroups: ["argoproj.io"], Resources: [ "analysistemplates", "clusteranalysistemplates", ], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "list", "watch", "update", "patch", "delete"], APIGroups: ["apps"], Resources: ["replicasets"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["get", "list", "watch", "update", "patch"], APIGroups: ["", "apps"], Resources: ["deployments", "podtemplates"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["get", "list", "watch", "patch", "create", "delete"], APIGroups: [""], Resources: ["services"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "update"], APIGroups: [ "coordination.k8s.io", ], Resources: ["leases"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["get", "list", "watch"], APIGroups: [""], Resources: ["secrets", "configmaps"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["list", "update", "watch"], APIGroups: [""], Resources: ["pods"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create"], APIGroups: [""], Resources: ["pods/eviction"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "update", "patch"], APIGroups: [""], Resources: ["events"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "list", "watch", "patch"], APIGroups: ["networking.k8s.io", "extensions"], Resources: ["ingresses"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "list", "watch", "update", "patch", "delete"], APIGroups: ["batch"], Resources: ["jobs"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["watch", "get", "update", "patch", "list"], APIGroups: [ "networking.istio.io", ], Resources: ["virtualservices", "destinationrules"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "watch", "get", "update", "patch"], APIGroups: ["split.smi-spec.io"], Resources: ["trafficsplits"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "watch", "get", "update", "list", "delete"], APIGroups: ["getambassador.io", "x.getambassador.io"], Resources: ["mappings", "... Gomega truncated this representation as it exceeds 'format.MaxLength'. Consider having the object provide a custom 'GomegaStringer' representation or adjust the parameters in Gomega's 'format' package. Learn more here: https://onsi.github.io/gomega/#adjusting-output to consist of <[]v1.PolicyRule | len:23, cap:23>: [ { Verbs: ["get", "list", "watch", "update", "patch"], APIGroups: ["argoproj.io"], Resources: [ "rollouts", "rollouts/status", "rollouts/finalizers", ], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "list", "watch", "update", "patch", "delete"], APIGroups: ["argoproj.io"], Resources: [ "analysisruns", "analysisruns/finalizers", "experiments", "experiments/finalizers", ], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["get", "list", "watch"], APIGroups: ["argoproj.io"], Resources: [ "analysistemplates", "clusteranalysistemplates", ], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "list", "watch", "update", "patch", "delete"], APIGroups: ["apps"], Resources: ["replicasets"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["get", "list", "watch"], APIGroups: ["", "apps"], Resources: ["deployments", "podtemplates"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["get", "list", "watch", "patch", "create", "delete"], APIGroups: [""], Resources: ["services"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "update"], APIGroups: [ "coordination.k8s.io", ], Resources: ["leases"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["get", "list", "watch"], APIGroups: [""], Resources: ["secrets", "configmaps"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["list", "update", "watch"], APIGroups: [""], Resources: ["pods"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create"], APIGroups: [""], Resources: ["pods/eviction"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "update", "patch"], APIGroups: [""], Resources: ["events"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "list", "watch", "patch"], APIGroups: ["networking.k8s.io", "extensions"], Resources: ["ingresses"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "get", "list", "watch", "update", "patch", "delete"], APIGroups: ["batch"], Resources: ["jobs"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["watch", "get", "update", "patch", "list"], APIGroups: [ "networking.istio.io", ], Resources: ["virtualservices", "destinationrules"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "watch", "get", "update", "patch"], APIGroups: ["split.smi-spec.io"], Resources: ["trafficsplits"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["create", "watch", "get", "update", "list", "delete"], APIGroups: ["getambassador.io", "x.getambassador.io"], Resources: ["mappings", "ambassadormappings"... Gomega truncated this representation as it exceeds 'format.MaxLength'. Consider having the object provide a custom 'GomegaStringer' representation or adjust the parameters in Gomega's 'format' package. Learn more here: https://onsi.github.io/gomega/#adjusting-output the missing elements were <[]v1.PolicyRule | len:2, cap:2>: [ { Verbs: ["get", "list", "watch"], APIGroups: ["", "apps"], Resources: ["deployments", "podtemplates"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["watch", "get", "update"], APIGroups: [ "traefik.containo.us", ], Resources: ["traefikservices"], ResourceNames: nil, NonResourceURLs: nil, }, ] the extra elements were <[]v1.PolicyRule | len:2, cap:2>: [ { Verbs: ["get", "list", "watch", "update", "patch"], APIGroups: ["", "apps"], Resources: ["deployments", "podtemplates"], ResourceNames: nil, NonResourceURLs: nil, }, { Verbs: ["watch", "get", "update"], APIGroups: [ "traefik.containo.us", "traefik.io", ], Resources: ["traefikservices"], ResourceNames: nil, NonResourceURLs: nil, }, ] In [It] at: /tmp/tmp.TxJP1GNlai/argo-rollouts-manager/tests/e2e/test_utils.go:215 @ 11/27/24 10:58:51.126 ------------------------------ HavePhase: expected: Available actual: HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Available HaveSuccessCondition: expected: {Reconciled True 0 0001-01-01 00:00:00 +0000 UTC Success } actual: {Reconciled True 0 2024-11-27 10:59:16 +0000 UTC Success } Object exists in ExistByName: rollout-bluegreen-active Object exists in ExistByName: rollout-bluegreen-preview HavePhase: expected: Failure actual: Failure HaveCondition: expected: {Reconciled False 0 0001-01-01 00:00:00 +0000 UTC InvalidRolloutManagerScope when Subscription has environment variable NAMESPACE_SCOPED_ARGO_ROLLOUTS set to False, there may not exist any namespace-scoped RolloutManagers: only a single cluster-scoped RolloutManager is supported} actual: {Reconciled False 0 2024-11-27 10:59:35 +0000 UTC InvalidRolloutManagerScope when Subscription has environment variable NAMESPACE_SCOPED_ARGO_ROLLOUTS set to False, there may not exist any namespace-scoped RolloutManagers: only a single cluster-scoped RolloutManager is supported} Object exists in ExistByName: rollout-bluegreen-active Object exists in ExistByName: rollout-bluegreen-preview HavePhase: expected: Available actual: Available HaveSuccessCondition: expected: {Reconciled True 0 0001-01-01 00:00:00 +0000 UTC Success } actual: {Reconciled True 0 2024-11-27 10:59:16 +0000 UTC Success } Object exists in ExistByName: rollout-bluegreen-active Object exists in ExistByName: rollout-bluegreen-preview •HavePhase: expected: Available actual: HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Available HaveSuccessCondition: expected: {Reconciled True 0 0001-01-01 00:00:00 +0000 UTC Success } actual: {Reconciled True 0 2024-11-27 11:00:25 +0000 UTC Success } Object exists in ExistByName: rollout-bluegreen-active Object exists in ExistByName: rollout-bluegreen-preview HavePhase: expected: Failure actual: Failure HaveCondition: expected: {Reconciled False 0 0001-01-01 00:00:00 +0000 UTC MultipleClusterScopedRolloutManager when there exists a cluster-scoped RolloutManager on the cluster, there may not exist another: only a single cluster-scoped RolloutManager is supported} actual: {Reconciled False 0 2024-11-27 11:00:43 +0000 UTC MultipleClusterScopedRolloutManager when there exists a cluster-scoped RolloutManager on the cluster, there may not exist another: only a single cluster-scoped RolloutManager is supported} Object exists in ExistByName: rollout-bluegreen-active Object exists in ExistByName: rollout-bluegreen-preview HavePhase: expected: Failure actual: Failure HaveCondition: expected: {Reconciled False 0 0001-01-01 00:00:00 +0000 UTC MultipleClusterScopedRolloutManager when there exists a cluster-scoped RolloutManager on the cluster, there may not exist another: only a single cluster-scoped RolloutManager is supported} actual: {Reconciled False 0 2024-11-27 11:00:43 +0000 UTC MultipleClusterScopedRolloutManager when there exists a cluster-scoped RolloutManager on the cluster, there may not exist another: only a single cluster-scoped RolloutManager is supported} Object exists in ExistByName: rollout-bluegreen-active Object exists in ExistByName: rollout-bluegreen-preview •HavePhase: expected: Failure actual: Failure HaveCondition: expected: {Reconciled False 0 0001-01-01 00:00:00 +0000 UTC InvalidRolloutManagerNamespace Namespace is not specified in CLUSTER_SCOPED_ARGO_ROLLOUTS_NAMESPACES environment variable of Subscription resource. If you wish to install a cluster-scoped Argo Rollouts instance outside the default namespace, ensure it is defined in CLUSTER_SCOPED_ARGO_ROLLOUTS_NAMESPACES} actual: {Reconciled False 0 2024-11-27 11:01:28 +0000 UTC InvalidRolloutManagerNamespace Namespace is not specified in CLUSTER_SCOPED_ARGO_ROLLOUTS_NAMESPACES environment variable of Subscription resource. If you wish to install a cluster-scoped Argo Rollouts instance outside the default namespace, ensure it is defined in CLUSTER_SCOPED_ARGO_ROLLOUTS_NAMESPACES} •HavePhase: expected: Available actual: HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Available • Summarizing 2 Failures: [FAIL] Cluster-scoped RolloutManager tests Testing cluster-scoped RolloutManager behaviour [It] After creating cluster-scoped RolloutManager in one namespace, operator should create appropriate K8s resources and watch argo rollouts CR in another namespace. /tmp/tmp.TxJP1GNlai/argo-rollouts-manager/tests/e2e/test_utils.go:215 [FAIL] Cluster-scoped RolloutManager tests Testing cluster-scoped RolloutManager behaviour [It] After creating cluster-scoped RolloutManager in a namespace, operator should create appropriate K8s resources and watch argo rollouts CR in multiple namespace. /tmp/tmp.TxJP1GNlai/argo-rollouts-manager/tests/e2e/test_utils.go:215 Ran 6 of 6 Specs in 256.692 seconds FAIL! -- 4 Passed | 2 Failed | 0 Pending | 0 Skipped --- FAIL: TestClusterScoped (256.69s) FAIL coverage: [no statements] FAIL github.com/argoproj-labs/argo-rollouts-manager/tests/e2e/cluster-scoped 256.806s FAIL make: *** [Makefile:120: test-e2e] Error 1 + exit_code=2 + [[ 2 != 0 ]] + echo 2 + tee /tekton/results/result 2 + failed=2 + exit_code=0 + kubectl delete rolloutmanagers --all -n test-rom-ns-1 No resources found + cd /tmp/tmp.TxJP1GNlai/argo-rollouts-manager + SKIP_RUN_STEP=true + hack/run-upstream-argo-rollouts-e2e-tests.sh + set -e + trap cleanup EXIT +++ dirname -- hack/run-upstream-argo-rollouts-e2e-tests.sh ++ cd -- hack ++ pwd + SCRIPT_DIR=/tmp/tmp.TxJP1GNlai/argo-rollouts-manager/hack ++ mktemp -d + TMP_DIR=/tmp/tmp.aCMO1twgu4 + cd /tmp/tmp.aCMO1twgu4 + git clone https://github.com/argoproj/argo-rollouts Cloning into 'argo-rollouts'... + cd argo-rollouts + git checkout v1.6.6 Note: switching to 'v1.6.6'. You are in 'detached HEAD' state. You can look around, make experimental changes and commit them, and you can discard any commits you make in this state without impacting any branches by switching back to a branch. If you want to create a new branch to retain commits you create, you may do so (now or later) by using -c with the switch command. Example: git switch -c Or undo this operation with: git switch - Turn off this advice by setting config variable advice.detachedHead to false HEAD is now at 737ca89b4 fix: stuck rollout when 2nd deployment happens before 1st finishes (#3354) + go mod tidy go: downloading github.com/blang/semver v3.5.1+incompatible go: downloading k8s.io/apiextensions-apiserver v0.25.8 go: downloading k8s.io/apimachinery v0.25.8 go: downloading k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading k8s.io/client-go v0.25.8 go: downloading k8s.io/cli-runtime v0.25.8 go: downloading k8s.io/klog/v2 v2.80.1 go: downloading github.com/influxdata/influxdb-client-go/v2 v2.12.3 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading k8s.io/code-generator v0.25.8 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/aws/aws-sdk-go-v2/config v1.19.1 go: downloading github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.27.2 go: downloading github.com/aws/aws-sdk-go-v2 v1.21.2 go: downloading k8s.io/api v0.25.8 go: downloading github.com/argoproj/pkg v0.13.6 go: downloading github.com/servicemeshinterface/smi-sdk-go v0.5.0 go: downloading github.com/spf13/cobra v1.7.0 go: downloading github.com/argoproj/notifications-engine v0.4.1-0.20231011160156-2d2d1a75dbee go: downloading k8s.io/kubernetes v1.25.8 go: downloading github.com/hashicorp/go-plugin v1.4.10 go: downloading github.com/tj/assert v0.0.3 go: downloading github.com/machinebox/graphql v0.2.2 go: downloading github.com/spaceapegames/go-wavefront v1.8.1 go: downloading github.com/newrelic/newrelic-client-go v1.1.0 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 go: downloading google.golang.org/grpc v1.57.0 go: downloading google.golang.org/protobuf v1.31.0 go: downloading google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 go: downloading k8s.io/component-base v0.25.8 go: downloading k8s.io/kubectl v0.25.8 go: downloading github.com/juju/ansiterm v1.0.0 go: downloading github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.21.0 go: downloading github.com/prometheus/common v0.42.0 go: downloading github.com/prometheus/common/sigv4 v0.1.0 go: downloading github.com/mitchellh/mapstructure v1.5.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/bombsimon/logrusr/v4 v4.0.0 go: downloading github.com/antonmedv/expr v1.15.3 go: downloading github.com/prometheus/client_model v0.3.0 go: downloading golang.org/x/sys v0.10.0 go: downloading github.com/golang/mock v1.6.0 go: downloading golang.org/x/net v0.12.0 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/stretchr/objx v0.5.0 go: downloading k8s.io/gengo v0.0.0-20220902162205-c0856e24416d go: downloading github.com/golang/glog v1.1.0 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.43 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 go: downloading github.com/aws/smithy-go v1.15.0 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/go-openapi/jsonreference v0.20.0 go: downloading github.com/go-openapi/swag v0.21.1 go: downloading github.com/google/gnostic v0.6.9 go: downloading github.com/inconshreveable/mousetrap v1.1.0 go: downloading golang.org/x/tools v0.6.0 go: downloading github.com/Azure/go-autorest/autorest v0.11.27 go: downloading github.com/Azure/go-autorest v14.2.0+incompatible go: downloading github.com/Azure/go-autorest/autorest/adal v0.9.20 go: downloading k8s.io/apiserver v0.25.8 go: downloading github.com/valyala/fasttemplate v1.2.2 go: downloading golang.org/x/term v0.10.0 go: downloading github.com/felixge/httpsnoop v1.0.3 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 go: downloading github.com/deepmap/oapi-codegen v1.11.0 go: downloading github.com/matryer/is v1.4.0 go: downloading github.com/hashicorp/go-hclog v0.14.1 go: downloading github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb go: downloading github.com/mitchellh/go-testing-interface v1.0.0 go: downloading github.com/oklog/run v1.0.0 go: downloading github.com/emicklei/go-restful/v3 v3.10.1 go: downloading github.com/mattn/go-colorable v0.1.12 go: downloading github.com/mattn/go-isatty v0.0.14 go: downloading github.com/aws/aws-sdk-go v1.44.116 go: downloading gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c go: downloading github.com/jhump/protoreflect v1.6.0 go: downloading github.com/elazarl/goproxy v0.0.0-20220417044921-416226498f94 go: downloading github.com/PagerDuty/go-pagerduty v1.7.0 go: downloading github.com/RocketChat/Rocket.Chat.Go.SDK v0.0.0-20220708192748-b73dcb041214 go: downloading github.com/aws/aws-sdk-go-v2/service/sqs v1.20.0 go: downloading github.com/bradleyfalzon/ghinstallation/v2 v2.5.0 go: downloading github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 go: downloading github.com/google/go-github/v41 v41.0.0 go: downloading github.com/gregdel/pushover v1.2.1 go: downloading github.com/opsgenie/opsgenie-go-sdk-v2 v1.2.13 go: downloading github.com/slack-go/slack v0.12.2 go: downloading github.com/spf13/cast v1.5.1 go: downloading github.com/whilp/git-urls v0.0.0-20191001220047-6db9661140c0 go: downloading gomodules.xyz/notify v0.1.1 go: downloading google.golang.org/api v0.132.0 go: downloading github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf go: downloading sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 go: downloading sigs.k8s.io/kustomize/api v0.12.1 go: downloading github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de go: downloading github.com/peterbourgon/diskv v2.0.1+incompatible go: downloading sigs.k8s.io/kustomize/kyaml v0.13.9 go: downloading golang.org/x/text v0.11.0 go: downloading github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 go: downloading github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 go: downloading github.com/go-openapi/jsonpointer v0.19.5 go: downloading github.com/jmespath/go-jmespath/internal/testify v1.5.1 go: downloading github.com/Azure/go-autorest/logger v0.2.1 go: downloading github.com/Azure/go-autorest/tracing v0.6.0 go: downloading github.com/Azure/go-autorest/autorest/mocks v0.4.2 go: downloading github.com/Azure/go-autorest/autorest/date v0.3.0 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading golang.org/x/crypto v0.11.0 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading google.golang.org/appengine v1.6.7 go: downloading cloud.google.com/go/compute v1.20.1 go: downloading github.com/valyala/bytebufferpool v1.0.0 go: downloading github.com/fatih/color v1.7.0 go: downloading github.com/blang/semver/v4 v4.0.0 go: downloading github.com/google/go-querystring v1.1.0 go: downloading github.com/hashicorp/go-retryablehttp v0.7.1 go: downloading github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 go: downloading github.com/valyala/fastjson v1.6.3 go: downloading github.com/onsi/ginkgo/v2 v2.1.6 go: downloading github.com/onsi/gomega v1.20.1 go: downloading github.com/lunixbochs/vtclean v1.0.0 go: downloading github.com/Masterminds/sprig/v3 v3.2.3 go: downloading github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f go: downloading github.com/chai2010/gettext-go v1.0.2 go: downloading github.com/MakeNowJust/heredoc v1.0.0 go: downloading github.com/mitchellh/go-wordwrap v1.0.1 go: downloading github.com/russross/blackfriday v1.6.0 go: downloading github.com/google/go-github/v53 v53.0.0 go: downloading github.com/gorilla/websocket v1.5.0 go: downloading github.com/go-test/deep v1.0.4 go: downloading github.com/frankban/quicktest v1.14.4 go: downloading gomodules.xyz/envconfig v1.3.1-0.20190308184047-426f31af0d45 go: downloading gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df go: downloading k8s.io/component-helpers v0.25.8 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/hashicorp/go-cleanhttp v0.5.2 go: downloading github.com/jpillora/backoff v1.0.0 go: downloading github.com/Masterminds/goutils v1.1.1 go: downloading github.com/Masterminds/semver/v3 v3.2.0 go: downloading github.com/huandu/xstrings v1.3.3 go: downloading github.com/mitchellh/copystructure v1.2.0 go: downloading github.com/shopspring/decimal v1.2.0 go: downloading golang.org/x/mod v0.8.0 go: downloading github.com/onsi/ginkgo v1.14.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae go: downloading github.com/Jeffail/gabs v1.4.0 go: downloading github.com/gopackage/ddp v0.0.0-20170117053602-652027933df4 go: downloading github.com/sony/sonyflake v1.0.0 go: downloading k8s.io/cluster-bootstrap v0.25.8 go: downloading github.com/docker/distribution v2.8.1+incompatible go: downloading gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc go: downloading github.com/mitchellh/reflectwalk v1.0.2 go: downloading golang.org/x/sync v0.3.0 go: downloading github.com/creack/pty v1.1.11 go: downloading gotest.tools/v3 v3.0.3 go: downloading github.com/go-errors/errors v1.4.2 go: downloading github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 go: downloading github.com/xlab/treeprint v1.1.0 go: downloading github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 go: downloading github.com/moby/spdystream v0.2.0 go: downloading github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 go: downloading go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 go: downloading github.com/sergi/go-diff v1.1.0 go: downloading github.com/cloudflare/circl v1.3.3 go: downloading github.com/opencontainers/go-digest v1.0.0 go: downloading go.opencensus.io v0.24.0 go: downloading github.com/google/s2a-go v0.1.4 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.5 + kubectl delete ns argo-rollouts namespace "argo-rollouts" deleted + kubectl wait --timeout=5m --for=delete namespace/argo-rollouts + kubectl create ns argo-rollouts namespace/argo-rollouts created + kubectl config set-context --current --namespace=argo-rollouts Context "default/api-psi-417-ocp-gitops-qe-com:6443/kube:admin" modified. + cd /tmp/tmp.TxJP1GNlai/argo-rollouts-manager/hack/.. + '[' -z true ']' + cd /tmp/tmp.aCMO1twgu4/argo-rollouts + cat + kubectl apply -f /tmp/tmp.aCMO1twgu4/rollout-manager.yaml rolloutmanager.argoproj.io/argo-rollout created + echo '* Waiting for Argo Rollouts Deployment to exist' * Waiting for Argo Rollouts Deployment to exist + kubectl get -n argo-rollouts deployment/argo-rollouts NAME READY UP-TO-DATE AVAILABLE AGE argo-rollouts 0/1 1 0 0s + kubectl wait --for=condition=Available --timeout=10m -n argo-rollouts deployment/argo-rollouts deployment.apps/argo-rollouts condition met + kubectl apply -f test/e2e/crds customresourcedefinition.apiextensions.k8s.io/apisixroutes.apisix.apache.org configured customresourcedefinition.apiextensions.k8s.io/wasmplugins.extensions.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/destinationrules.networking.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/envoyfilters.networking.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/gateways.networking.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/proxyconfigs.networking.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/serviceentries.networking.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/sidecars.networking.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/virtualservices.networking.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/workloadentries.networking.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/workloadgroups.networking.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/authorizationpolicies.security.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/peerauthentications.security.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/requestauthentications.security.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/telemetries.telemetry.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/trafficsplits.split.smi-spec.io unchanged + oc adm policy add-scc-to-user anyuid -z argo-rollouts -n argo-rollouts clusterrole.rbac.authorization.k8s.io/system:openshift:scc:anyuid added: "argo-rollouts" + oc adm policy add-scc-to-user anyuid -z default -n argo-rollouts clusterrole.rbac.authorization.k8s.io/system:openshift:scc:anyuid added: "default" + rm -f /tmp/test-e2e.log + set +e + make test-e2e + tee /tmp/test-e2e.log ./hack/installers/install-dev-tools.sh +++ dirname ./hack/installers/install-dev-tools.sh ++ cd ./hack/installers/../.. ++ pwd + PROJECT_ROOT=/tmp/tmp.aCMO1twgu4/argo-rollouts + DIST_PATH=/tmp/tmp.aCMO1twgu4/argo-rollouts/dist + PATH=/tmp/tmp.aCMO1twgu4/argo-rollouts/dist:/tmp/tmp.aCMO1twgu4/argo-rollouts/dist:/root/.nvm/versions/node/v20.12.2/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin + mkdir -p /tmp/tmp.aCMO1twgu4/argo-rollouts/dist + gotestsum_version=1.10.1 ++ go env GOOS + OS=linux ++ go env GOARCH + ARCH=amd64 + export TARGET_FILE=gotestsum_1.10.1_linux_amd64.tar.gz + TARGET_FILE=gotestsum_1.10.1_linux_amd64.tar.gz + temp_path=/tmp/gotestsum_1.10.1_linux_amd64.tar.gz + url=https://github.com/gotestyourself/gotestsum/releases/download/v1.10.1/gotestsum_1.10.1_linux_amd64.tar.gz + '[' -e /tmp/gotestsum_1.10.1_linux_amd64.tar.gz ']' + curl -sLf --retry 3 -o /tmp/gotestsum_1.10.1_linux_amd64.tar.gz https://github.com/gotestyourself/gotestsum/releases/download/v1.10.1/gotestsum_1.10.1_linux_amd64.tar.gz + mkdir -p /tmp/gotestsum-1.10.1 + tar -xvzf /tmp/gotestsum_1.10.1_linux_amd64.tar.gz -C /tmp/gotestsum-1.10.1 LICENSE LICENSE.md README.md gotestsum + cp /tmp/gotestsum-1.10.1/gotestsum /tmp/tmp.aCMO1twgu4/argo-rollouts/dist/gotestsum + chmod +x /tmp/tmp.aCMO1twgu4/argo-rollouts/dist/gotestsum + gotestsum --version gotestsum version 1.10.1 /tmp/tmp.aCMO1twgu4/argo-rollouts/dist/gotestsum --rerun-fails-report=rerunreport.txt --junitfile=junit.xml --format=testname --packages="./test/e2e" --rerun-fails=5 -- -timeout 60m -count 1 --tags e2e -p 1 -parallel 1 -v --short ./test/e2e PASS test/e2e.TestAnalysisSuite/TestAnalysisWithArgs (39.63s) PASS test/e2e.TestAnalysisSuite/TestAnalysisWithSecret (29.99s) PASS test/e2e.TestAnalysisSuite/TestBackgroundAnalysisWithArgs (29.39s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenAbortAndUpdate (22.29s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenAnalysis (33.26s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenKitchenSink (47.03s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenPostPromotionFail (17.42s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenPrePromotionFail (28.40s) PASS test/e2e.TestAnalysisSuite/TestCanaryBackgroundAnalysis (4.72s) PASS test/e2e.TestAnalysisSuite/TestCanaryFailInlineMultipleAnalysis (12.03s) PASS test/e2e.TestAnalysisSuite/TestCanaryInlineAnalysis (18.09s) PASS test/e2e.TestAnalysisSuite/TestCanaryInlineMultipleAnalysis (17.33s) PASS test/e2e.TestAnalysisSuite/TestMultipleAnalysis (10.63s) PASS test/e2e.TestAnalysisSuite (319.32s) === RUN TestAPISIXSuite/TestAPISIXCanarySetHeaderStep time="2024-11-27T11:18:54Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:18:56Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-11-27T11:18:58Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-11-27T11:19:00Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route created\n" rollout=rollout-apisix-canary time="2024-11-27T11:19:02Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:19:02Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-11-27T11:19:02Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (7.90s) FAIL test/e2e.TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (7.90s) PASS test/e2e.TestAPISIXSuite/TestAPISIXCanaryStep (28.17s) === RUN TestAPISIXSuite --- FAIL: TestAPISIXSuite (36.42s) FAIL test/e2e.TestAPISIXSuite (36.42s) PASS test/e2e.TestAWSSuite/TestALBExperimentStep (23.73s) PASS test/e2e.TestAWSSuite/TestALBExperimentStepMultiIngress (24.17s) === RUN TestAWSSuite/TestALBExperimentStepNoSetWeight time="2024-11-27T11:20:20Z" level=info msg="Deleting e2e-test-name=TestALBExperimentStepNoSetWeight" time="2024-11-27T11:20:22Z" level=info msg="service/alb-rollout-root created\n" rollout=alb-rollout time="2024-11-27T11:20:25Z" level=info msg="service/alb-rollout-canary created\n" rollout=alb-rollout time="2024-11-27T11:20:27Z" level=info msg="service/alb-rollout-stable created\n" rollout=alb-rollout time="2024-11-27T11:20:28Z" level=info msg="Warning: annotation \"kubernetes.io/ingress.class\" is deprecated, please use 'spec.ingressClassName' instead\ningress.networking.k8s.io/alb-rollout-ingress created\n" rollout=alb-rollout time="2024-11-27T11:20:30Z" level=info msg="rollout.argoproj.io/alb-rollout created\n" rollout=alb-rollout time="2024-11-27T11:20:30Z" level=info msg="Waiting for condition: status=Healthy" rollout=alb-rollout time="2024-11-27T11:20:31Z" level=info msg="Condition 'status=Healthy' met after 1s" rollout=alb-rollout time="2024-11-27T11:20:31Z" level=info msg="Experiment expectation 'experiment count == 0' met" rollout=alb-rollout time="2024-11-27T11:20:31Z" level=info msg="Updated rollout pod spec: 2024-11-27T11:20:31.230562779Z" rollout=alb-rollout time="2024-11-27T11:20:31Z" level=info msg="Sleeping 10s" rollout=alb-rollout require.go:72: Error Trace: /root/go/pkg/mod/github.com/tj/assert@v0.0.3/require.go:72 /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/aws_test.go:264 /tmp/tmp.aCMO1twgu4/argo-rollouts/test/fixtures/then.go:28 /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/aws_test.go:254 Error: Not equal: expected: "{\"Type\":\"forward\",\"ForwardConfig\":{\"TargetGroups\":[{\"ServiceName\":\"alb-rollout-canary\",\"ServicePort\":\"80\",\"Weight\":0},{\"ServiceName\":\"\",\"ServicePort\":\"80\",\"Weight\":20},{\"ServiceName\":\"\",\"ServicePort\":\"80\",\"Weight\":20},{\"ServiceName\":\"alb-rollout-stable\",\"ServicePort\":\"80\",\"Weight\":60}]}}" actual : "{\"Type\":\"forward\",\"ForwardConfig\":{\"TargetGroups\":[{\"ServiceName\":\"alb-rollout-canary\",\"ServicePort\":\"80\",\"Weight\":0},{\"ServicePort\":\"80\",\"Weight\":20},{\"ServicePort\":\"80\",\"Weight\":20},{\"ServiceName\":\"alb-rollout-stable\",\"ServicePort\":\"80\",\"Weight\":60}]}}" Diff: --- Expected +++ Actual @@ -1 +1 @@ -{"Type":"forward","ForwardConfig":{"TargetGroups":[{"ServiceName":"alb-rollout-canary","ServicePort":"80","Weight":0},{"ServiceName":"","ServicePort":"80","Weight":20},{"ServiceName":"","ServicePort":"80","Weight":20},{"ServiceName":"alb-rollout-stable","ServicePort":"80","Weight":60}]}} +{"Type":"forward","ForwardConfig":{"TargetGroups":[{"ServiceName":"alb-rollout-canary","ServicePort":"80","Weight":0},{"ServicePort":"80","Weight":20},{"ServicePort":"80","Weight":20},{"ServiceName":"alb-rollout-stable","ServicePort":"80","Weight":60}]}} Test: TestAWSSuite/TestALBExperimentStepNoSetWeight Name: alb-rollout Namespace: argo-rollouts Status: ◌ Progressing Message: more replicas need to be updated Strategy: Canary Step: 0/1 SetWeight: 0 ActualWeight: 0 Images: nginx:1.19-alpine (stable, Σ:experiment-alb-canary, Σ:experiment-alb-stable) Replicas: Desired: 1 Current: 1 Updated: 0 Ready: 1 Available: 1 NAME KIND STATUS AGE INFO ⟳ alb-rollout Rollout ◌ Progressing 11s ├──# revision:2 │ ├──⧉ alb-rollout-bbf848ddb ReplicaSet • ScaledDown 10s canary │ └──Σ alb-rollout-bbf848ddb-2-0 Experiment ◌ Running 10s │ ├──⧉ alb-rollout-bbf848ddb-2-0-experiment-alb-canary ReplicaSet ✔ Healthy 10s │ │ └──□ alb-rollout-bbf848ddb-2-0-experiment-alb-canary-zx7s9 Pod ✔ Running 10s ready:1/1 │ └──⧉ alb-rollout-bbf848ddb-2-0-experiment-alb-stable ReplicaSet ✔ Healthy 10s │ └──□ alb-rollout-bbf848ddb-2-0-experiment-alb-stable-s78xv Pod ✔ Running 10s ready:1/1 └──# revision:1 └──⧉ alb-rollout-d6f4fdcdc ReplicaSet ✔ Healthy 11s stable └──□ alb-rollout-d6f4fdcdc-lzj4x Pod ✔ Running 11s ready:1/1 --- apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: annotations: rollout.argoproj.io/revision: "2" creationTimestamp: "2024-11-27T11:20:30Z" generation: 3 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestALBExperimentStepNoSetWeight name: alb-rollout namespace: argo-rollouts resourceVersion: "603391" uid: a644917c-fe3e-4d42-91bc-f57fb2120ef4 spec: replicas: 1 selector: matchLabels: app: alb-rollout strategy: canary: canaryService: alb-rollout-canary stableService: alb-rollout-stable steps: - experiment: duration: 15s templates: - metadata: {} name: experiment-alb-canary specRef: canary weight: 20 - metadata: {} name: experiment-alb-stable specRef: stable weight: 20 trafficRouting: alb: ingress: alb-rollout-ingress rootService: alb-rollout-root servicePort: 80 template: metadata: annotations: update: "2024-11-27T11:20:31.230562779Z" creationTimestamp: null labels: app: alb-rollout spec: containers: - image: nginx:1.19-alpine name: alb-rollout ports: - containerPort: 80 name: http protocol: TCP resources: requests: cpu: 5m memory: 16Mi status: HPAReplicas: 1 availableReplicas: 1 blueGreen: {} canary: currentExperiment: alb-rollout-bbf848ddb-2-0 weights: additional: - weight: 20 - weight: 20 canary: podTemplateHash: bbf848ddb serviceName: alb-rollout-canary weight: 0 stable: podTemplateHash: d6f4fdcdc serviceName: alb-rollout-stable weight: 60 conditions: - lastTransitionTime: "2024-11-27T11:20:31Z" lastUpdateTime: "2024-11-27T11:20:31Z" message: Rollout has minimum availability reason: AvailableReason status: "True" type: Available - lastTransitionTime: "2024-11-27T11:20:31Z" lastUpdateTime: "2024-11-27T11:20:31Z" message: Rollout is not healthy reason: RolloutHealthy status: "False" type: Healthy - lastTransitionTime: "2024-11-27T11:20:31Z" lastUpdateTime: "2024-11-27T11:20:31Z" message: RolloutCompleted reason: RolloutCompleted status: "False" type: Completed - lastTransitionTime: "2024-11-27T11:20:30Z" lastUpdateTime: "2024-11-27T11:20:31Z" message: Created new replica set "alb-rollout-bbf848ddb" reason: NewReplicaSetCreated status: "True" type: Progressing currentPodHash: bbf848ddb currentStepHash: 8b59978 currentStepIndex: 0 message: more replicas need to be updated observedGeneration: "3" phase: Progressing readyReplicas: 1 replicas: 1 selector: app=alb-rollout stableRS: d6f4fdcdc 2024-11-27T11:20:30Z Normal RolloutAddedToInformer Rollout resource added to informer: argo-rollouts/alb-rollout 2024-11-27T11:20:30Z Normal RolloutNotCompleted Rollout not completed, started update to revision 1 (d6f4fdcdc) 2024-11-27T11:20:30Z Normal RolloutUpdated Rollout updated to revision 1 2024-11-27T11:20:30Z Normal NewReplicaSetCreated Created ReplicaSet alb-rollout-d6f4fdcdc (revision 1) 2024-11-27T11:20:32Z (x2) Normal PatchingALBIngress Updating Ingress `alb-rollout-ingress` to desiredWeight '0' 2024-11-27T11:20:30Z Normal TrafficWeightUpdated Traffic weight updated to 0 2024-11-27T11:20:30Z Normal ScalingReplicaSet Scaled up ReplicaSet alb-rollout-d6f4fdcdc (revision 1) from 0 to 1 2024-11-27T11:20:30Z Normal RolloutCompleted Rollout completed update to revision 1 (d6f4fdcdc): Initial deploy 2024-11-27T11:20:31Z (x2) Normal TrafficWeightUpdated Traffic weight updated 2024-11-27T11:20:31Z Normal SwitchService Switched selector for service 'alb-rollout-stable' from '' to 'd6f4fdcdc' 2024-11-27T11:20:31Z Normal SwitchService Switched selector for service 'alb-rollout-canary' from '' to 'd6f4fdcdc' 2024-11-27T11:20:31Z Normal RolloutNotCompleted Rollout not completed, started update to revision 2 (bbf848ddb) 2024-11-27T11:20:31Z Normal RolloutUpdated Rollout updated to revision 2 2024-11-27T11:20:31Z Normal NewReplicaSetCreated Created ReplicaSet alb-rollout-bbf848ddb (revision 2) 2024-11-27T11:20:31Z Normal ExperimentCreated Created Experiment 'alb-rollout-bbf848ddb-2-0' 2024-11-27T11:20:32Z Normal TrafficWeightUpdated Traffic weight updated additional: [{20 } {20 }] time="2024-11-27T11:20:41Z" level=info msg="Deleting e2e-test-name=TestALBExperimentStepNoSetWeight" --- FAIL: TestAWSSuite/TestALBExperimentStepNoSetWeight (21.39s) FAIL test/e2e.TestAWSSuite/TestALBExperimentStepNoSetWeight (21.39s) PASS test/e2e.TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress (25.82s) PASS test/e2e.TestAWSSuite/TestALBPingPongUpdate (17.98s) PASS test/e2e.TestAWSSuite/TestALBPingPongUpdateMultiIngress (17.98s) PASS test/e2e.TestAWSSuite/TestAlbHeaderRoute (14.40s) PASS test/e2e.TestAWSSuite/TestAlbHeaderRouteMultiIngress (27.08s) === RUN TestAWSSuite --- FAIL: TestAWSSuite (174.69s) FAIL test/e2e.TestAWSSuite (174.69s) PASS test/e2e.TestBlueGreenSuite/TestBlueGreenPreviewReplicaCount (8.45s) PASS test/e2e.TestBlueGreenSuite/TestBlueGreenProgressDeadlineExceededWithPause (64.93s) PASS test/e2e.TestBlueGreenSuite/TestBlueGreenProgressDeadlineExceededWithoutPause (65.42s) PASS test/e2e.TestBlueGreenSuite/TestEphemeralMetadata (11.21s) PASS test/e2e.TestBlueGreenSuite (150.35s) PASS test/e2e.TestCanarySuite/TestCanaryDynamicStableScale (27.20s) PASS test/e2e.TestCanarySuite/TestCanaryDynamicStableScaleRollbackToStable (21.18s) PASS test/e2e.TestCanarySuite/TestCanaryProgressDeadlineExceededWithPause (124.53s) PASS test/e2e.TestCanarySuite/TestCanaryScaleDownDelay (13.59s) === RUN TestCanarySuite/TestCanaryScaleDownOnAbort time="2024-11-27T11:28:04Z" level=info msg="Deleting e2e-test-name=TestCanaryScaleDownOnAbort" time="2024-11-27T11:28:05Z" level=info msg="service/canary-scaledowndelay-root created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:06Z" level=info msg="service/canary-scaledowndelay-canary created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:07Z" level=info msg="service/canary-scaledowndelay-stable created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:08Z" level=info msg="Warning: annotation \"kubernetes.io/ingress.class\" is deprecated, please use 'spec.ingressClassName' instead\ningress.networking.k8s.io/canary-scaledowndelay-ingress created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:09Z" level=info msg="rollout.argoproj.io/canary-scaledownd-on-abort created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:09Z" level=info msg="Waiting for condition: status=Healthy" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:10Z" level=info msg="Condition 'status=Healthy' met after 1s" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:10Z" level=info msg="Updated rollout pod spec: 2024-11-27T11:28:10.646641521Z" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:10Z" level=info msg="Waiting for condition: status=Paused" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:11Z" level=info msg="Condition 'status=Paused' met after 0s" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:11Z" level=info msg="Aborted rollout" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:11Z" level=info msg="Waiting for condition: status=Degraded" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:11Z" level=info msg="Condition 'status=Degraded' met after 0s" rollout=canary-scaledownd-on-abort canary_test.go:544: Expected canary-scaledowndelay-canary selector: map[app:canary-scaledowndelay rollouts-pod-template-hash:66597877b7]. Actual: map[app:canary-scaledowndelay rollouts-pod-template-hash:6f47467c87] Name: canary-scaledownd-on-abort Namespace: argo-rollouts Status: ✖ Degraded Message: RolloutAborted: Rollout aborted update to revision 2 Strategy: Canary Step: 0/2 SetWeight: 0 ActualWeight: 0 Images: nginx:1.19-alpine (canary, stable) Replicas: Desired: 1 Current: 2 Updated: 1 Ready: 2 Available: 2 NAME KIND STATUS AGE INFO ⟳ canary-scaledownd-on-abort Rollout ✖ Degraded 2s ├──# revision:2 │ └──⧉ canary-scaledownd-on-abort-6f47467c87 ReplicaSet ✔ Healthy 1s canary,delay:0s │ └──□ canary-scaledownd-on-abort-6f47467c87-4vn5m Pod ✔ Running 1s ready:1/1 └──# revision:1 └──⧉ canary-scaledownd-on-abort-66597877b7 ReplicaSet ✔ Healthy 2s stable └──□ canary-scaledownd-on-abort-66597877b7-7t4ns Pod ✔ Running 2s ready:1/1 --- apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: annotations: rollout.argoproj.io/revision: "2" creationTimestamp: "2024-11-27T11:28:09Z" generation: 3 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestCanaryScaleDownOnAbort name: canary-scaledownd-on-abort namespace: argo-rollouts resourceVersion: "609836" uid: 1e3ee3bd-9aec-4268-914b-a8a78bbabbd8 spec: replicas: 1 selector: matchLabels: app: canary-scaledowndelay strategy: canary: abortScaleDownDelaySeconds: 1 canaryService: canary-scaledowndelay-canary scaleDownDelayRevisionLimit: 1 stableService: canary-scaledowndelay-stable steps: - setCanaryScale: replicas: 1 - pause: {} trafficRouting: alb: ingress: canary-scaledowndelay-ingress rootService: canary-scaledowndelay-root servicePort: 80 template: metadata: annotations: update: "2024-11-27T11:28:10.646641521Z" creationTimestamp: null labels: app: canary-scaledowndelay spec: containers: - image: nginx:1.19-alpine name: canary-scaledowndelay ports: - containerPort: 80 name: http protocol: TCP resources: requests: cpu: 5m memory: 16Mi status: HPAReplicas: 2 abort: true abortedAt: "2024-11-27T11:28:11Z" availableReplicas: 2 blueGreen: {} canary: weights: canary: podTemplateHash: 6f47467c87 serviceName: canary-scaledowndelay-canary weight: 0 stable: podTemplateHash: 66597877b7 serviceName: canary-scaledowndelay-stable weight: 100 conditions: - lastTransitionTime: "2024-11-27T11:28:10Z" lastUpdateTime: "2024-11-27T11:28:10Z" message: Rollout has minimum availability reason: AvailableReason status: "True" type: Available - lastTransitionTime: "2024-11-27T11:28:10Z" lastUpdateTime: "2024-11-27T11:28:10Z" message: Rollout is not healthy reason: RolloutHealthy status: "False" type: Healthy - lastTransitionTime: "2024-11-27T11:28:10Z" lastUpdateTime: "2024-11-27T11:28:10Z" message: RolloutCompleted reason: RolloutCompleted status: "False" type: Completed - lastTransitionTime: "2024-11-27T11:28:11Z" lastUpdateTime: "2024-11-27T11:28:11Z" message: Rollout is paused reason: RolloutPaused status: "True" type: Paused - lastTransitionTime: "2024-11-27T11:28:11Z" lastUpdateTime: "2024-11-27T11:28:11Z" message: Rollout aborted update to revision 2 reason: RolloutAborted status: "False" type: Progressing currentPodHash: 6f47467c87 currentStepHash: 5db8f9b897 currentStepIndex: 0 message: 'RolloutAborted: Rollout aborted update to revision 2' observedGeneration: "3" phase: Degraded readyReplicas: 2 replicas: 2 selector: app=canary-scaledowndelay stableRS: 66597877b7 updatedReplicas: 1 2024-11-27T11:28:09Z Normal RolloutAddedToInformer Rollout resource added to informer: argo-rollouts/canary-scaledownd-on-abort 2024-11-27T11:28:09Z Normal RolloutNotCompleted Rollout not completed, started update to revision 1 (66597877b7) 2024-11-27T11:28:09Z Normal RolloutUpdated Rollout updated to revision 1 2024-11-27T11:28:09Z Normal NewReplicaSetCreated Created ReplicaSet canary-scaledownd-on-abort-66597877b7 (revision 1) 2024-11-27T11:28:09Z Normal PatchingALBIngress Updating Ingress `canary-scaledowndelay-ingress` to desiredWeight '0' 2024-11-27T11:28:09Z Normal TrafficWeightUpdated Traffic weight updated to 0 2024-11-27T11:28:09Z Normal ScalingReplicaSet Scaled up ReplicaSet canary-scaledownd-on-abort-66597877b7 (revision 1) from 0 to 1 2024-11-27T11:28:09Z Normal RolloutCompleted Rollout completed update to revision 1 (66597877b7): Initial deploy 2024-11-27T11:28:10Z (x2) Normal TrafficWeightUpdated Traffic weight updated 2024-11-27T11:28:10Z Normal SwitchService Switched selector for service 'canary-scaledowndelay-stable' from '' to '66597877b7' 2024-11-27T11:28:10Z Normal SwitchService Switched selector for service 'canary-scaledowndelay-canary' from '' to '66597877b7' 2024-11-27T11:28:10Z Normal RolloutNotCompleted Rollout not completed, started update to revision 2 (6f47467c87) 2024-11-27T11:28:10Z Normal RolloutUpdated Rollout updated to revision 2 2024-11-27T11:28:10Z Normal NewReplicaSetCreated Created ReplicaSet canary-scaledownd-on-abort-6f47467c87 (revision 2) 2024-11-27T11:28:10Z Normal ScalingReplicaSet Scaled up ReplicaSet canary-scaledownd-on-abort-6f47467c87 (revision 2) from 0 to 1 2024-11-27T11:28:11Z (x3) Normal SwitchService Switched selector for service 'canary-scaledowndelay-canary' from '66597877b7' to '6f47467c87' 2024-11-27T11:28:11Z Normal RolloutStepCompleted Rollout step 1/2 completed (setCanaryScale{replicas: 1}) 2024-11-27T11:28:11Z Normal RolloutPaused Rollout is paused (CanaryPauseStep) 2024-11-27T11:28:11Z (x3) Normal SwitchService Switched selector for service 'canary-scaledowndelay-canary' from '6f47467c87' to '66597877b7' 2024-11-27T11:28:11Z Warning RolloutAborted Rollout aborted update to revision 2 time="2024-11-27T11:28:11Z" level=info msg="Deleting e2e-test-name=TestCanaryScaleDownOnAbort" --- FAIL: TestCanarySuite/TestCanaryScaleDownOnAbort (8.12s) FAIL test/e2e.TestCanarySuite/TestCanaryScaleDownOnAbort (8.12s) PASS test/e2e.TestCanarySuite/TestCanaryScaleDownOnAbortNoTrafficRouting (10.90s) PASS test/e2e.TestCanarySuite/TestCanaryUnScaleDownOnAbort (10.74s) PASS test/e2e.TestCanarySuite/TestCanaryWithPausedRollout (101.70s) PASS test/e2e.TestCanarySuite/TestEphemeralMetadata (7.58s) PASS test/e2e.TestCanarySuite/TestReduceWeightAndHonorMaxUnavailable (28.20s) PASS test/e2e.TestCanarySuite/TestRolloutScalingDuringUpdate (3.19s) PASS test/e2e.TestCanarySuite/TestRolloutScalingWhenPaused (5.78s) PASS test/e2e.TestCanarySuite/TestRolloutWithMaxSurgeScalingDuringUpdate (3.29s) PASS test/e2e.TestCanarySuite/TestCanarySetCanaryScale (12.75s) === RUN TestCanarySuite time="2024-11-27T11:24:57Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" --- FAIL: TestCanarySuite (368.30s) FAIL test/e2e.TestCanarySuite (368.30s) === RUN TestExperimentSuite/TestExperimentWithDryRunMetrics time="2024-11-27T11:31:16Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:31:18Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-11-27T11:31:19Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-11-27T11:31:19Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-11-27T11:31:50Z" level=info msg="Condition 'phase=Successful' met after 30s" time="2024-11-27T11:31:50Z" level=info msg="Sleeping 3s" time="2024-11-27T11:31:53Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-11-27T11:31:53Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-11-27T11:31:53Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 34s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 34s delay:26s │ └──□ experiment-with-dry-run-baseline-hx7fs Pod ✔ Running 34s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ 8b5825f3-7def-4baf-a1c9-dc27d4d283ef.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-11-27T11:31:19Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "614161" uid: c009d82c-b28d-4b7c-87e8-33fd16ea091c spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-11-27T11:31:20Z" conditions: - lastTransitionTime: "2024-11-27T11:31:50Z" lastUpdateTime: "2024-11-27T11:31:50Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-11-27T11:31:50Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-11-27T11:31:19Z (x2) Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-11-27T11:31:19Z (x2) Normal ExperimentPending Experiment transitioned from -> Pending 2024-11-27T11:31:19Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-11-27T11:31:20Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-11-27T11:31:20Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-11-27T11:31:20Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-11-27T11:31:20Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-11-27T11:31:50Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-11-27T11:31:50Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-11-27T11:31:50Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-11-27T11:31:53Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (36.76s) FAIL test/e2e.TestExperimentSuite/TestExperimentWithDryRunMetrics (36.76s) PASS test/e2e.TestExperimentSuite/TestExperimentWithMeasurementRetentionMetrics (36.80s) PASS test/e2e.TestExperimentSuite/TestExperimentWithMultiportServiceAndScaleDownDelay (18.06s) PASS test/e2e.TestExperimentSuite/TestExperimentWithServiceAndScaleDownDelay (18.03s) PASS test/e2e.TestExperimentSuite/TestExperimentWithServiceNameAndScaleDownDelay (17.97s) PASS test/e2e.TestExperimentSuite/TestRolloutWithExperimentAndAnalysis (10.25s) === RUN TestExperimentSuite --- FAIL: TestExperimentSuite (128.26s) FAIL test/e2e.TestExperimentSuite (128.26s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenExceedProgressDeadlineAbort (11.46s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenPreviewReplicaCountPromoteFull (9.55s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenPromoteFull (8.07s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenScaleDownDelay (10.35s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenScaleDownOnAbort (11.94s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenToCanary (6.92s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenUpdate (7.08s) PASS test/e2e.TestFunctionalSuite/TestCanaryPromoteFull (8.91s) PASS test/e2e.TestFunctionalSuite/TestCompleteRolloutRestart (21.82s) PASS test/e2e.TestFunctionalSuite/TestContainerResourceFormats (4.25s) === RUN TestFunctionalSuite/TestControllerMetrics time="2024-11-27T11:35:17Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-11-27T11:35:17Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 6133 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc001c8c9c0, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc000d83c00) /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc000b16680?, 0xc0006733c0?, 0xc0017d8660?}, {0x3212d41, 0x4}, {0xc001503e70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc000b16680?, 0xc0006733c0?, 0xc000d83c00?}, {0xc001503e70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc001c8c9c0) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc001c8c9c0, 0xc00112b4d0) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 4875 /usr/local/go/src/testing/testing.go:1648 +0x3ad --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.66s) FAIL test/e2e.TestFunctionalSuite/TestControllerMetrics (0.66s) PASS test/e2e.TestFunctionalSuite/TestFixInvalidSpec (22.50s) PASS test/e2e.TestFunctionalSuite/TestKubectlWaitForCompleted (24.63s) PASS test/e2e.TestFunctionalSuite/TestKubectlWaitForPaused (7.75s) PASS test/e2e.TestFunctionalSuite/TestMalformedRollout (3.66s) PASS test/e2e.TestFunctionalSuite/TestMalformedRolloutEphemeralCtr (3.84s) PASS test/e2e.TestFunctionalSuite/TestRolloutAbortRetryPromote (10.04s) PASS test/e2e.TestFunctionalSuite/TestRolloutBadToBadUpdate (23.98s) PASS test/e2e.TestFunctionalSuite/TestRolloutBadToGoodSetWeightUpdate (22.39s) PASS test/e2e.TestFunctionalSuite/TestRolloutBadToGoodUpdate (17.07s) PASS test/e2e.TestFunctionalSuite/TestRolloutGoodToBadUpdate (25.50s) PASS test/e2e.TestFunctionalSuite/TestRolloutPDBRestart (48.78s) PASS test/e2e.TestFunctionalSuite/TestRolloutPauseDurationGreaterThanProgressDeadlineSeconds (37.68s) PASS test/e2e.TestFunctionalSuite/TestRolloutPodVolumesItemsSpec (11.35s) PASS test/e2e.TestFunctionalSuite/TestRolloutRestart (18.14s) PASS test/e2e.TestFunctionalSuite/TestWorkloadRef (19.58s) PASS test/e2e.TestFunctionalSuite/TestWorkloadRefTemplate (4.33s) === RUN TestFunctionalSuite time="2024-11-27T11:33:37Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" --- FAIL: TestFunctionalSuite (404.63s) FAIL test/e2e.TestFunctionalSuite (404.63s) PASS test/e2e.TestHeaderRoutingSuite/TestIstioHostHeaderRoute (19.48s) PASS test/e2e.TestHeaderRoutingSuite (19.81s) PASS test/e2e.TestIstioSuite/TestIstioAbortUpdate (11.25s) PASS test/e2e.TestIstioSuite/TestIstioAbortUpdateDeleteAllCanaryPods (11.87s) PASS test/e2e.TestIstioSuite/TestIstioHostSplit (22.62s) PASS test/e2e.TestIstioSuite/TestIstioHostSplitExperimentStep (21.04s) PASS test/e2e.TestIstioSuite/TestIstioHostSplitOnlyTls (11.10s) PASS test/e2e.TestIstioSuite/TestIstioSubsetSplit (13.72s) PASS test/e2e.TestIstioSuite/TestIstioSubsetSplitExperimentStep (20.27s) PASS test/e2e.TestIstioSuite/TestIstioSubsetSplitSingleRoute (10.49s) PASS test/e2e.TestIstioSuite/TestIstioUpdateInMiddleZeroCanaryReplicas (11.19s) PASS test/e2e.TestIstioSuite (133.89s) PASS test/e2e.TestMirrorRouteSuite/TestIstioHostMirrorRoute (17.23s) PASS test/e2e.TestMirrorRouteSuite (17.57s) PASS test/e2e.TestRollbackSuite/TestRollbackAnalysisOutsideWindow (31.50s) PASS test/e2e.TestRollbackSuite/TestRollbackAnalysisWithinWindow (24.01s) PASS test/e2e.TestRollbackSuite (60.34s) FAIL test/e2e DONE 104 tests, 6 skipped, 10 failures in 2444.701s === RUN TestAPISIXSuite/TestAPISIXCanarySetHeaderStep time="2024-11-27T11:44:27Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:44:29Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-11-27T11:44:31Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-11-27T11:44:33Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route configured\n" rollout=rollout-apisix-canary time="2024-11-27T11:44:35Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:44:35Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-11-27T11:44:35Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:44:36Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (8.39s) FAIL test/e2e.TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 1) (8.39s) === RUN TestAPISIXSuite --- FAIL: TestAPISIXSuite (8.77s) FAIL test/e2e.TestAPISIXSuite (re-run 1) (8.77s) FAIL test/e2e PASS test/e2e.TestAWSSuite/TestALBExperimentStepNoSetWeight (re-run 1) (23.92s) PASS test/e2e.TestAWSSuite (re-run 1) (24.30s) PASS test/e2e === RUN TestCanarySuite/TestCanaryScaleDownOnAbort time="2024-11-27T11:45:22Z" level=info msg="Deleting e2e-test-name=TestCanaryScaleDownOnAbort" time="2024-11-27T11:45:23Z" level=info msg="service/canary-scaledowndelay-root created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:25Z" level=info msg="service/canary-scaledowndelay-canary created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:27Z" level=info msg="service/canary-scaledowndelay-stable created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:28Z" level=info msg="Warning: annotation \"kubernetes.io/ingress.class\" is deprecated, please use 'spec.ingressClassName' instead\ningress.networking.k8s.io/canary-scaledowndelay-ingress created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:30Z" level=info msg="rollout.argoproj.io/canary-scaledownd-on-abort created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:30Z" level=info msg="Waiting for condition: status=Healthy" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:31Z" level=info msg="Condition 'status=Healthy' met after 0s" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:31Z" level=info msg="Updated rollout pod spec: 2024-11-27T11:45:31.101663566Z" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:31Z" level=info msg="Waiting for condition: status=Paused" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:32Z" level=info msg="Condition 'status=Paused' met after 0s" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:32Z" level=info msg="Aborted rollout" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:32Z" level=info msg="Waiting for condition: status=Degraded" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:32Z" level=info msg="Condition 'status=Degraded' met after 0s" rollout=canary-scaledownd-on-abort canary_test.go:544: Expected canary-scaledowndelay-canary selector: map[app:canary-scaledowndelay rollouts-pod-template-hash:66597877b7]. Actual: map[app:canary-scaledowndelay rollouts-pod-template-hash:f87547876] Name: canary-scaledownd-on-abort Namespace: argo-rollouts Status: ✖ Degraded Message: RolloutAborted: Rollout aborted update to revision 2 Strategy: Canary Step: 0/2 SetWeight: 0 ActualWeight: 0 Images: nginx:1.19-alpine (canary, stable) Replicas: Desired: 1 Current: 2 Updated: 1 Ready: 2 Available: 2 NAME KIND STATUS AGE INFO ⟳ canary-scaledownd-on-abort Rollout ✖ Degraded 2s ├──# revision:2 │ └──⧉ canary-scaledownd-on-abort-f87547876 ReplicaSet ✔ Healthy 1s canary,delay:0s │ └──□ canary-scaledownd-on-abort-f87547876-fr7dj Pod ✔ Running 1s ready:1/1 └──# revision:1 └──⧉ canary-scaledownd-on-abort-66597877b7 ReplicaSet ✔ Healthy 2s stable └──□ canary-scaledownd-on-abort-66597877b7-qj7jq Pod ✔ Running 2s ready:1/1 --- apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: annotations: rollout.argoproj.io/revision: "2" creationTimestamp: "2024-11-27T11:45:30Z" generation: 3 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestCanaryScaleDownOnAbort name: canary-scaledownd-on-abort namespace: argo-rollouts resourceVersion: "627668" uid: 124c499d-86e4-467e-ac90-e7edc70fa588 spec: replicas: 1 selector: matchLabels: app: canary-scaledowndelay strategy: canary: abortScaleDownDelaySeconds: 1 canaryService: canary-scaledowndelay-canary scaleDownDelayRevisionLimit: 1 stableService: canary-scaledowndelay-stable steps: - setCanaryScale: replicas: 1 - pause: {} trafficRouting: alb: ingress: canary-scaledowndelay-ingress rootService: canary-scaledowndelay-root servicePort: 80 template: metadata: annotations: update: "2024-11-27T11:45:31.101663566Z" creationTimestamp: null labels: app: canary-scaledowndelay spec: containers: - image: nginx:1.19-alpine name: canary-scaledowndelay ports: - containerPort: 80 name: http protocol: TCP resources: requests: cpu: 5m memory: 16Mi status: HPAReplicas: 2 abort: true abortedAt: "2024-11-27T11:45:32Z" availableReplicas: 2 blueGreen: {} canary: weights: canary: podTemplateHash: f87547876 serviceName: canary-scaledowndelay-canary weight: 0 stable: podTemplateHash: 66597877b7 serviceName: canary-scaledowndelay-stable weight: 100 conditions: - lastTransitionTime: "2024-11-27T11:45:31Z" lastUpdateTime: "2024-11-27T11:45:31Z" message: Rollout has minimum availability reason: AvailableReason status: "True" type: Available - lastTransitionTime: "2024-11-27T11:45:31Z" lastUpdateTime: "2024-11-27T11:45:31Z" message: Rollout is not healthy reason: RolloutHealthy status: "False" type: Healthy - lastTransitionTime: "2024-11-27T11:45:31Z" lastUpdateTime: "2024-11-27T11:45:31Z" message: RolloutCompleted reason: RolloutCompleted status: "False" type: Completed - lastTransitionTime: "2024-11-27T11:45:32Z" lastUpdateTime: "2024-11-27T11:45:32Z" message: Rollout is paused reason: RolloutPaused status: "True" type: Paused - lastTransitionTime: "2024-11-27T11:45:32Z" lastUpdateTime: "2024-11-27T11:45:32Z" message: Rollout aborted update to revision 2 reason: RolloutAborted status: "False" type: Progressing currentPodHash: f87547876 currentStepHash: 5db8f9b897 currentStepIndex: 0 message: 'RolloutAborted: Rollout aborted update to revision 2' observedGeneration: "3" phase: Degraded readyReplicas: 2 replicas: 2 selector: app=canary-scaledowndelay stableRS: 66597877b7 updatedReplicas: 1 2024-11-27T11:45:30Z Normal RolloutAddedToInformer Rollout resource added to informer: argo-rollouts/canary-scaledownd-on-abort 2024-11-27T11:45:30Z Normal RolloutNotCompleted Rollout not completed, started update to revision 1 (66597877b7) 2024-11-27T11:45:30Z Normal RolloutUpdated Rollout updated to revision 1 2024-11-27T11:45:30Z Normal NewReplicaSetCreated Created ReplicaSet canary-scaledownd-on-abort-66597877b7 (revision 1) 2024-11-27T11:45:30Z Normal PatchingALBIngress Updating Ingress `canary-scaledowndelay-ingress` to desiredWeight '0' 2024-11-27T11:45:30Z Normal TrafficWeightUpdated Traffic weight updated to 0 2024-11-27T11:45:30Z Normal ScalingReplicaSet Scaled up ReplicaSet canary-scaledownd-on-abort-66597877b7 (revision 1) from 0 to 1 2024-11-27T11:45:30Z Normal RolloutCompleted Rollout completed update to revision 1 (66597877b7): Initial deploy 2024-11-27T11:45:31Z (x2) Normal TrafficWeightUpdated Traffic weight updated 2024-11-27T11:45:31Z Normal SwitchService Switched selector for service 'canary-scaledowndelay-stable' from '' to '66597877b7' 2024-11-27T11:45:31Z Normal SwitchService Switched selector for service 'canary-scaledowndelay-canary' from '' to '66597877b7' 2024-11-27T11:45:31Z Normal RolloutNotCompleted Rollout not completed, started update to revision 2 (f87547876) 2024-11-27T11:45:31Z Normal RolloutUpdated Rollout updated to revision 2 2024-11-27T11:45:31Z Normal NewReplicaSetCreated Created ReplicaSet canary-scaledownd-on-abort-f87547876 (revision 2) 2024-11-27T11:45:31Z Normal ScalingReplicaSet Scaled up ReplicaSet canary-scaledownd-on-abort-f87547876 (revision 2) from 0 to 1 2024-11-27T11:45:32Z (x3) Normal SwitchService Switched selector for service 'canary-scaledowndelay-canary' from '66597877b7' to 'f87547876' 2024-11-27T11:45:32Z Normal RolloutStepCompleted Rollout step 1/2 completed (setCanaryScale{replicas: 1}) 2024-11-27T11:45:32Z Normal RolloutPaused Rollout is paused (CanaryPauseStep) 2024-11-27T11:45:32Z (x3) Normal SwitchService Switched selector for service 'canary-scaledowndelay-canary' from 'f87547876' to '66597877b7' 2024-11-27T11:45:32Z Warning RolloutAborted Rollout aborted update to revision 2 time="2024-11-27T11:45:32Z" level=info msg="Deleting e2e-test-name=TestCanaryScaleDownOnAbort" time="2024-11-27T11:45:32Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestCanarySuite/TestCanaryScaleDownOnAbort (10.72s) FAIL test/e2e.TestCanarySuite/TestCanaryScaleDownOnAbort (re-run 1) (10.72s) === RUN TestCanarySuite time="2024-11-27T11:45:22Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" --- FAIL: TestCanarySuite (12.93s) FAIL test/e2e.TestCanarySuite (re-run 1) (12.93s) FAIL test/e2e === RUN TestExperimentSuite/TestExperimentWithDryRunMetrics time="2024-11-27T11:45:42Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:45:44Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-11-27T11:45:46Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-11-27T11:45:46Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-11-27T11:46:18Z" level=info msg="Condition 'phase=Successful' met after 31s" time="2024-11-27T11:46:18Z" level=info msg="Sleeping 3s" time="2024-11-27T11:46:21Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-11-27T11:46:21Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-11-27T11:46:21Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 35s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 35s delay:26s │ └──□ experiment-with-dry-run-baseline-jspbf Pod ✔ Running 35s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ 05828cdb-f0b1-4bdb-8c1c-b989222baf50.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-11-27T11:45:46Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "628184" uid: c417c693-29ab-4e9d-ba7b-d0e858fd4b95 spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-11-27T11:45:48Z" conditions: - lastTransitionTime: "2024-11-27T11:46:18Z" lastUpdateTime: "2024-11-27T11:46:18Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-11-27T11:46:18Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-11-27T11:45:46Z (x2) Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-11-27T11:45:46Z (x2) Normal ExperimentPending Experiment transitioned from -> Pending 2024-11-27T11:45:46Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-11-27T11:45:48Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-11-27T11:45:48Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-11-27T11:45:48Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-11-27T11:45:48Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-11-27T11:46:18Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-11-27T11:46:18Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-11-27T11:46:18Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-11-27T11:46:21Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:46:21Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (38.63s) FAIL test/e2e.TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 1) (38.63s) === RUN TestExperimentSuite --- FAIL: TestExperimentSuite (38.98s) FAIL test/e2e.TestExperimentSuite (re-run 1) (38.98s) FAIL test/e2e === RUN TestFunctionalSuite/TestControllerMetrics time="2024-11-27T11:46:33Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-11-27T11:46:33Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 95 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc0005009c0, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc0007bfa00) /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc00080ed40?, 0xc000a06f40?, 0xc000308320?}, {0x3212d41, 0x4}, {0xc000c1de70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc00080ed40?, 0xc000a06f40?, 0xc0007bfa00?}, {0xc000c1de70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc0005009c0) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc0005009c0, 0xc000c2ba70) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 82 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-11-27T11:46:34Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.54s) FAIL test/e2e.TestFunctionalSuite/TestControllerMetrics (re-run 1) (0.54s) === RUN TestFunctionalSuite time="2024-11-27T11:46:33Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" --- FAIL: TestFunctionalSuite (2.76s) FAIL test/e2e.TestFunctionalSuite (re-run 1) (2.76s) FAIL test/e2e DONE 2 runs, 114 tests, 6 skipped, 18 failures in 2587.341s === RUN TestAPISIXSuite/TestAPISIXCanarySetHeaderStep time="2024-11-27T11:46:44Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:46:46Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-11-27T11:46:48Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-11-27T11:46:50Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:46:52Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:46:52Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-11-27T11:46:52Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:46:53Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (8.69s) FAIL test/e2e.TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 2) (8.69s) === RUN TestAPISIXSuite --- FAIL: TestAPISIXSuite (9.04s) FAIL test/e2e.TestAPISIXSuite (re-run 2) (9.04s) FAIL test/e2e PASS test/e2e.TestCanarySuite/TestCanaryScaleDownOnAbort (re-run 2) (14.67s) PASS test/e2e.TestCanarySuite (re-run 2) (16.94s) PASS test/e2e === RUN TestExperimentSuite/TestExperimentWithDryRunMetrics time="2024-11-27T11:47:30Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:47:32Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-11-27T11:47:34Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-11-27T11:47:34Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-11-27T11:48:05Z" level=info msg="Condition 'phase=Successful' met after 30s" time="2024-11-27T11:48:05Z" level=info msg="Sleeping 3s" time="2024-11-27T11:48:08Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-11-27T11:48:08Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-11-27T11:48:08Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 34s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 34s delay:26s │ └──□ experiment-with-dry-run-baseline-nmhcq Pod ✔ Running 34s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ 37841be3-5dab-4086-aae9-d6d925618b61.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-11-27T11:47:34Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "629336" uid: ec99d88d-3ee7-4561-9573-78d054e953d7 spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-11-27T11:47:35Z" conditions: - lastTransitionTime: "2024-11-27T11:48:05Z" lastUpdateTime: "2024-11-27T11:48:05Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-11-27T11:48:05Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-11-27T11:47:34Z Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-11-27T11:47:34Z Normal ExperimentPending Experiment transitioned from -> Pending 2024-11-27T11:47:34Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-11-27T11:47:35Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-11-27T11:47:35Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-11-27T11:47:35Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-11-27T11:47:35Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-11-27T11:48:05Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-11-27T11:48:05Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-11-27T11:48:05Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-11-27T11:48:08Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:48:08Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (38.36s) FAIL test/e2e.TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 2) (38.36s) === RUN TestExperimentSuite --- FAIL: TestExperimentSuite (38.76s) FAIL test/e2e.TestExperimentSuite (re-run 2) (38.76s) FAIL test/e2e === RUN TestFunctionalSuite/TestControllerMetrics time="2024-11-27T11:48:20Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-11-27T11:48:20Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 46 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc000103520, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc00052a800) /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc0003136c0?, 0xc0009361d8?, 0xc0005543a0?}, {0x3212d41, 0x4}, {0xc000819e70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc0003136c0?, 0xc0009361d8?, 0xc00052a800?}, {0xc000819e70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc000103520) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc000103520, 0xc000910f30) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 71 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-11-27T11:48:20Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.52s) FAIL test/e2e.TestFunctionalSuite/TestControllerMetrics (re-run 2) (0.52s) === RUN TestFunctionalSuite time="2024-11-27T11:48:20Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" --- FAIL: TestFunctionalSuite (2.50s) FAIL test/e2e.TestFunctionalSuite (re-run 2) (2.50s) FAIL test/e2e DONE 3 runs, 122 tests, 6 skipped, 24 failures in 2693.910s === RUN TestAPISIXSuite/TestAPISIXCanarySetHeaderStep time="2024-11-27T11:48:30Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:48:32Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-11-27T11:48:34Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-11-27T11:48:36Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:48:37Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:48:37Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-11-27T11:48:37Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:48:38Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (7.50s) FAIL test/e2e.TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 3) (7.50s) === RUN TestAPISIXSuite --- FAIL: TestAPISIXSuite (7.86s) FAIL test/e2e.TestAPISIXSuite (re-run 3) (7.86s) FAIL test/e2e === RUN TestExperimentSuite/TestExperimentWithDryRunMetrics time="2024-11-27T11:48:48Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:48:50Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-11-27T11:48:52Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-11-27T11:48:52Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-11-27T11:49:23Z" level=info msg="Condition 'phase=Successful' met after 30s" time="2024-11-27T11:49:23Z" level=info msg="Sleeping 3s" time="2024-11-27T11:49:26Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-11-27T11:49:26Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-11-27T11:49:26Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 34s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 34s delay:26s │ └──□ experiment-with-dry-run-baseline-lvlbd Pod ✔ Running 34s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ 71ef3d10-ad82-411d-8084-62ca2d5e76d3.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-11-27T11:48:52Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "629956" uid: 403a5f20-b6ab-4f6c-a5d1-9710712a91d8 spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-11-27T11:48:53Z" conditions: - lastTransitionTime: "2024-11-27T11:49:23Z" lastUpdateTime: "2024-11-27T11:49:23Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-11-27T11:49:23Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-11-27T11:48:52Z Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-11-27T11:48:52Z Normal ExperimentPending Experiment transitioned from -> Pending 2024-11-27T11:48:52Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-11-27T11:48:53Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-11-27T11:48:53Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-11-27T11:48:53Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-11-27T11:48:53Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-11-27T11:49:23Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-11-27T11:49:23Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-11-27T11:49:23Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-11-27T11:49:26Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:49:26Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (37.98s) FAIL test/e2e.TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 3) (37.98s) === RUN TestExperimentSuite --- FAIL: TestExperimentSuite (38.40s) FAIL test/e2e.TestExperimentSuite (re-run 3) (38.40s) FAIL test/e2e === RUN TestFunctionalSuite/TestControllerMetrics time="2024-11-27T11:49:38Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-11-27T11:49:38Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 95 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc000c824e0, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc00050db00) /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc000d2cac0?, 0xc000d0d378?, 0xc000122160?}, {0x3212d41, 0x4}, {0xc0003bfe70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc000d2cac0?, 0xc000d0d378?, 0xc00050db00?}, {0xc0003bfe70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc000c824e0) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc000c824e0, 0xc0002f5050) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 28 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-11-27T11:49:39Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.54s) FAIL test/e2e.TestFunctionalSuite/TestControllerMetrics (re-run 3) (0.54s) === RUN TestFunctionalSuite time="2024-11-27T11:49:38Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" --- FAIL: TestFunctionalSuite (2.74s) FAIL test/e2e.TestFunctionalSuite (re-run 3) (2.74s) FAIL test/e2e DONE 4 runs, 128 tests, 6 skipped, 30 failures in 2772.338s === RUN TestAPISIXSuite/TestAPISIXCanarySetHeaderStep time="2024-11-27T11:49:49Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:49:51Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-11-27T11:49:53Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-11-27T11:49:54Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:49:56Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:49:56Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-11-27T11:49:56Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:49:56Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (7.53s) FAIL test/e2e.TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 4) (7.53s) === RUN TestAPISIXSuite --- FAIL: TestAPISIXSuite (7.87s) FAIL test/e2e.TestAPISIXSuite (re-run 4) (7.87s) FAIL test/e2e === RUN TestExperimentSuite/TestExperimentWithDryRunMetrics time="2024-11-27T11:50:06Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:50:08Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-11-27T11:50:10Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-11-27T11:50:10Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-11-27T11:50:41Z" level=info msg="Condition 'phase=Successful' met after 30s" time="2024-11-27T11:50:41Z" level=info msg="Sleeping 3s" time="2024-11-27T11:50:44Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-11-27T11:50:44Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-11-27T11:50:44Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 34s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 34s delay:26s │ └──□ experiment-with-dry-run-baseline-h2qq5 Pod ✔ Running 34s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ d1e3a87a-7f20-47b8-b2ff-035c046de2d9.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-11-27T11:50:10Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "630574" uid: 43d916ec-745f-4c99-811d-37e90729f98a spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-11-27T11:50:11Z" conditions: - lastTransitionTime: "2024-11-27T11:50:41Z" lastUpdateTime: "2024-11-27T11:50:41Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-11-27T11:50:41Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-11-27T11:50:10Z (x2) Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-11-27T11:50:10Z (x2) Normal ExperimentPending Experiment transitioned from -> Pending 2024-11-27T11:50:10Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-11-27T11:50:11Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-11-27T11:50:11Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-11-27T11:50:11Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-11-27T11:50:11Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-11-27T11:50:41Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-11-27T11:50:41Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-11-27T11:50:41Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-11-27T11:50:44Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:50:44Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (37.90s) FAIL test/e2e.TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 4) (37.90s) === RUN TestExperimentSuite --- FAIL: TestExperimentSuite (38.22s) FAIL test/e2e.TestExperimentSuite (re-run 4) (38.22s) FAIL test/e2e === RUN TestFunctionalSuite/TestControllerMetrics time="2024-11-27T11:50:57Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-11-27T11:50:57Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 40 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc000b6e000, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc0006eb000) /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc0004fd300?, 0xc000131638?, 0xc000b06fe0?}, {0x3212d41, 0x4}, {0xc00086be70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc0004fd300?, 0xc000131638?, 0xc0006eb000?}, {0xc00086be70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc000b6e000) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc000b6e000, 0xc000b1a5a0) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 82 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-11-27T11:50:58Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.57s) FAIL test/e2e.TestFunctionalSuite/TestControllerMetrics (re-run 4) (0.57s) === RUN TestFunctionalSuite time="2024-11-27T11:50:57Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" --- FAIL: TestFunctionalSuite (3.65s) FAIL test/e2e.TestFunctionalSuite (re-run 4) (3.65s) FAIL test/e2e DONE 5 runs, 134 tests, 6 skipped, 36 failures in 2851.237s === RUN TestAPISIXSuite/TestAPISIXCanarySetHeaderStep time="2024-11-27T11:51:08Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:51:12Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-11-27T11:51:14Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-11-27T11:51:16Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:51:18Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:51:18Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-11-27T11:51:18Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:51:20Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (11.43s) FAIL test/e2e.TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 5) (11.43s) === RUN TestAPISIXSuite --- FAIL: TestAPISIXSuite (14.51s) FAIL test/e2e.TestAPISIXSuite (re-run 5) (14.51s) FAIL test/e2e === RUN TestExperimentSuite/TestExperimentWithDryRunMetrics time="2024-11-27T11:51:32Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:51:38Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-11-27T11:51:46Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-11-27T11:51:46Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-11-27T11:52:17Z" level=info msg="Condition 'phase=Successful' met after 30s" time="2024-11-27T11:52:17Z" level=info msg="Sleeping 3s" time="2024-11-27T11:52:20Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-11-27T11:52:20Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-11-27T11:52:20Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 34s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 34s delay:26s │ └──□ experiment-with-dry-run-baseline-4nxmx Pod ✔ Running 34s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ cc10b951-1dcd-4a60-94a7-1b8bb0b154b2.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-11-27T11:51:46Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "631410" uid: 98aa4cfb-165f-4c93-8158-9ccddf6d4109 spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-11-27T11:51:47Z" conditions: - lastTransitionTime: "2024-11-27T11:52:17Z" lastUpdateTime: "2024-11-27T11:52:17Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-11-27T11:52:17Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-11-27T11:51:46Z (x2) Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-11-27T11:51:46Z (x2) Normal ExperimentPending Experiment transitioned from -> Pending 2024-11-27T11:51:46Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-11-27T11:51:47Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-11-27T11:51:47Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-11-27T11:51:47Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-11-27T11:51:47Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-11-27T11:52:17Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-11-27T11:52:17Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-11-27T11:52:17Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-11-27T11:52:20Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:52:20Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (47.79s) FAIL test/e2e.TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 5) (47.79s) === RUN TestExperimentSuite --- FAIL: TestExperimentSuite (48.14s) FAIL test/e2e.TestExperimentSuite (re-run 5) (48.14s) FAIL test/e2e === RUN TestFunctionalSuite/TestControllerMetrics time="2024-11-27T11:52:33Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-11-27T11:52:33Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 89 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc000602d00, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc0001b3100) /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc000810840?, 0xc00061e1b0?, 0xc000c903d0?}, {0x3212d41, 0x4}, {0xc000c03e70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc000810840?, 0xc00061e1b0?, 0xc0001b3100?}, {0xc000c03e70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc000602d00) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc000602d00, 0xc000b1e5a0) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 82 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-11-27T11:52:33Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.61s) FAIL test/e2e.TestFunctionalSuite/TestControllerMetrics (re-run 5) (0.61s) === RUN TestFunctionalSuite time="2024-11-27T11:52:33Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" --- FAIL: TestFunctionalSuite (3.49s) FAIL test/e2e.TestFunctionalSuite (re-run 5) (3.49s) FAIL test/e2e === Skipped === SKIP: test/e2e TestAppMeshSuite (0.05s) === SKIP: test/e2e TestAWSSuite/TestALBBlueGreenUpdate (0.58s) time="2024-11-27T11:19:30Z" level=info msg="Deleting e2e-test-name=TestALBBlueGreenUpdate" time="2024-11-27T11:19:31Z" level=info msg="Deleting e2e-test-name=TestALBBlueGreenUpdate" --- SKIP: TestAWSSuite/TestALBBlueGreenUpdate (0.58s) === SKIP: test/e2e TestAWSSuite/TestALBCanaryUpdate (0.60s) time="2024-11-27T11:19:31Z" level=info msg="Deleting e2e-test-name=TestALBCanaryUpdate" time="2024-11-27T11:19:31Z" level=info msg="Deleting e2e-test-name=TestALBCanaryUpdate" --- SKIP: TestAWSSuite/TestALBCanaryUpdate (0.60s) === SKIP: test/e2e TestAWSSuite/TestALBCanaryUpdateMultiIngress (0.60s) time="2024-11-27T11:19:32Z" level=info msg="Deleting e2e-test-name=TestALBCanaryUpdateMultiIngress" time="2024-11-27T11:19:32Z" level=info msg="Deleting e2e-test-name=TestALBCanaryUpdateMultiIngress" --- SKIP: TestAWSSuite/TestALBCanaryUpdateMultiIngress (0.60s) === SKIP: test/e2e TestSMIIngressSuite (0.05s) === SKIP: test/e2e TestSMISuite (0.05s) === Failed === FAIL: test/e2e TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (7.90s) time="2024-11-27T11:18:54Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:18:56Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-11-27T11:18:58Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-11-27T11:19:00Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route created\n" rollout=rollout-apisix-canary time="2024-11-27T11:19:02Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:19:02Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-11-27T11:19:02Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (7.90s) === FAIL: test/e2e TestAPISIXSuite (36.42s) === FAIL: test/e2e TestAWSSuite/TestALBExperimentStepNoSetWeight (21.39s) time="2024-11-27T11:20:20Z" level=info msg="Deleting e2e-test-name=TestALBExperimentStepNoSetWeight" time="2024-11-27T11:20:22Z" level=info msg="service/alb-rollout-root created\n" rollout=alb-rollout time="2024-11-27T11:20:25Z" level=info msg="service/alb-rollout-canary created\n" rollout=alb-rollout time="2024-11-27T11:20:27Z" level=info msg="service/alb-rollout-stable created\n" rollout=alb-rollout time="2024-11-27T11:20:28Z" level=info msg="Warning: annotation \"kubernetes.io/ingress.class\" is deprecated, please use 'spec.ingressClassName' instead\ningress.networking.k8s.io/alb-rollout-ingress created\n" rollout=alb-rollout time="2024-11-27T11:20:30Z" level=info msg="rollout.argoproj.io/alb-rollout created\n" rollout=alb-rollout time="2024-11-27T11:20:30Z" level=info msg="Waiting for condition: status=Healthy" rollout=alb-rollout time="2024-11-27T11:20:31Z" level=info msg="Condition 'status=Healthy' met after 1s" rollout=alb-rollout time="2024-11-27T11:20:31Z" level=info msg="Experiment expectation 'experiment count == 0' met" rollout=alb-rollout time="2024-11-27T11:20:31Z" level=info msg="Updated rollout pod spec: 2024-11-27T11:20:31.230562779Z" rollout=alb-rollout time="2024-11-27T11:20:31Z" level=info msg="Sleeping 10s" rollout=alb-rollout require.go:72: Error Trace: /root/go/pkg/mod/github.com/tj/assert@v0.0.3/require.go:72 /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/aws_test.go:264 /tmp/tmp.aCMO1twgu4/argo-rollouts/test/fixtures/then.go:28 /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/aws_test.go:254 Error: Not equal: expected: "{\"Type\":\"forward\",\"ForwardConfig\":{\"TargetGroups\":[{\"ServiceName\":\"alb-rollout-canary\",\"ServicePort\":\"80\",\"Weight\":0},{\"ServiceName\":\"\",\"ServicePort\":\"80\",\"Weight\":20},{\"ServiceName\":\"\",\"ServicePort\":\"80\",\"Weight\":20},{\"ServiceName\":\"alb-rollout-stable\",\"ServicePort\":\"80\",\"Weight\":60}]}}" actual : "{\"Type\":\"forward\",\"ForwardConfig\":{\"TargetGroups\":[{\"ServiceName\":\"alb-rollout-canary\",\"ServicePort\":\"80\",\"Weight\":0},{\"ServicePort\":\"80\",\"Weight\":20},{\"ServicePort\":\"80\",\"Weight\":20},{\"ServiceName\":\"alb-rollout-stable\",\"ServicePort\":\"80\",\"Weight\":60}]}}" Diff: --- Expected +++ Actual @@ -1 +1 @@ -{"Type":"forward","ForwardConfig":{"TargetGroups":[{"ServiceName":"alb-rollout-canary","ServicePort":"80","Weight":0},{"ServiceName":"","ServicePort":"80","Weight":20},{"ServiceName":"","ServicePort":"80","Weight":20},{"ServiceName":"alb-rollout-stable","ServicePort":"80","Weight":60}]}} +{"Type":"forward","ForwardConfig":{"TargetGroups":[{"ServiceName":"alb-rollout-canary","ServicePort":"80","Weight":0},{"ServicePort":"80","Weight":20},{"ServicePort":"80","Weight":20},{"ServiceName":"alb-rollout-stable","ServicePort":"80","Weight":60}]}} Test: TestAWSSuite/TestALBExperimentStepNoSetWeight Name: alb-rollout Namespace: argo-rollouts Status: ◌ Progressing Message: more replicas need to be updated Strategy: Canary Step: 0/1 SetWeight: 0 ActualWeight: 0 Images: nginx:1.19-alpine (stable, Σ:experiment-alb-canary, Σ:experiment-alb-stable) Replicas: Desired: 1 Current: 1 Updated: 0 Ready: 1 Available: 1 NAME KIND STATUS AGE INFO ⟳ alb-rollout Rollout ◌ Progressing 11s ├──# revision:2 │ ├──⧉ alb-rollout-bbf848ddb ReplicaSet • ScaledDown 10s canary │ └──Σ alb-rollout-bbf848ddb-2-0 Experiment ◌ Running 10s │ ├──⧉ alb-rollout-bbf848ddb-2-0-experiment-alb-canary ReplicaSet ✔ Healthy 10s │ │ └──□ alb-rollout-bbf848ddb-2-0-experiment-alb-canary-zx7s9 Pod ✔ Running 10s ready:1/1 │ └──⧉ alb-rollout-bbf848ddb-2-0-experiment-alb-stable ReplicaSet ✔ Healthy 10s │ └──□ alb-rollout-bbf848ddb-2-0-experiment-alb-stable-s78xv Pod ✔ Running 10s ready:1/1 └──# revision:1 └──⧉ alb-rollout-d6f4fdcdc ReplicaSet ✔ Healthy 11s stable └──□ alb-rollout-d6f4fdcdc-lzj4x Pod ✔ Running 11s ready:1/1 --- apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: annotations: rollout.argoproj.io/revision: "2" creationTimestamp: "2024-11-27T11:20:30Z" generation: 3 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestALBExperimentStepNoSetWeight name: alb-rollout namespace: argo-rollouts resourceVersion: "603391" uid: a644917c-fe3e-4d42-91bc-f57fb2120ef4 spec: replicas: 1 selector: matchLabels: app: alb-rollout strategy: canary: canaryService: alb-rollout-canary stableService: alb-rollout-stable steps: - experiment: duration: 15s templates: - metadata: {} name: experiment-alb-canary specRef: canary weight: 20 - metadata: {} name: experiment-alb-stable specRef: stable weight: 20 trafficRouting: alb: ingress: alb-rollout-ingress rootService: alb-rollout-root servicePort: 80 template: metadata: annotations: update: "2024-11-27T11:20:31.230562779Z" creationTimestamp: null labels: app: alb-rollout spec: containers: - image: nginx:1.19-alpine name: alb-rollout ports: - containerPort: 80 name: http protocol: TCP resources: requests: cpu: 5m memory: 16Mi status: HPAReplicas: 1 availableReplicas: 1 blueGreen: {} canary: currentExperiment: alb-rollout-bbf848ddb-2-0 weights: additional: - weight: 20 - weight: 20 canary: podTemplateHash: bbf848ddb serviceName: alb-rollout-canary weight: 0 stable: podTemplateHash: d6f4fdcdc serviceName: alb-rollout-stable weight: 60 conditions: - lastTransitionTime: "2024-11-27T11:20:31Z" lastUpdateTime: "2024-11-27T11:20:31Z" message: Rollout has minimum availability reason: AvailableReason status: "True" type: Available - lastTransitionTime: "2024-11-27T11:20:31Z" lastUpdateTime: "2024-11-27T11:20:31Z" message: Rollout is not healthy reason: RolloutHealthy status: "False" type: Healthy - lastTransitionTime: "2024-11-27T11:20:31Z" lastUpdateTime: "2024-11-27T11:20:31Z" message: RolloutCompleted reason: RolloutCompleted status: "False" type: Completed - lastTransitionTime: "2024-11-27T11:20:30Z" lastUpdateTime: "2024-11-27T11:20:31Z" message: Created new replica set "alb-rollout-bbf848ddb" reason: NewReplicaSetCreated status: "True" type: Progressing currentPodHash: bbf848ddb currentStepHash: 8b59978 currentStepIndex: 0 message: more replicas need to be updated observedGeneration: "3" phase: Progressing readyReplicas: 1 replicas: 1 selector: app=alb-rollout stableRS: d6f4fdcdc 2024-11-27T11:20:30Z Normal RolloutAddedToInformer Rollout resource added to informer: argo-rollouts/alb-rollout 2024-11-27T11:20:30Z Normal RolloutNotCompleted Rollout not completed, started update to revision 1 (d6f4fdcdc) 2024-11-27T11:20:30Z Normal RolloutUpdated Rollout updated to revision 1 2024-11-27T11:20:30Z Normal NewReplicaSetCreated Created ReplicaSet alb-rollout-d6f4fdcdc (revision 1) 2024-11-27T11:20:32Z (x2) Normal PatchingALBIngress Updating Ingress `alb-rollout-ingress` to desiredWeight '0' 2024-11-27T11:20:30Z Normal TrafficWeightUpdated Traffic weight updated to 0 2024-11-27T11:20:30Z Normal ScalingReplicaSet Scaled up ReplicaSet alb-rollout-d6f4fdcdc (revision 1) from 0 to 1 2024-11-27T11:20:30Z Normal RolloutCompleted Rollout completed update to revision 1 (d6f4fdcdc): Initial deploy 2024-11-27T11:20:31Z (x2) Normal TrafficWeightUpdated Traffic weight updated 2024-11-27T11:20:31Z Normal SwitchService Switched selector for service 'alb-rollout-stable' from '' to 'd6f4fdcdc' 2024-11-27T11:20:31Z Normal SwitchService Switched selector for service 'alb-rollout-canary' from '' to 'd6f4fdcdc' 2024-11-27T11:20:31Z Normal RolloutNotCompleted Rollout not completed, started update to revision 2 (bbf848ddb) 2024-11-27T11:20:31Z Normal RolloutUpdated Rollout updated to revision 2 2024-11-27T11:20:31Z Normal NewReplicaSetCreated Created ReplicaSet alb-rollout-bbf848ddb (revision 2) 2024-11-27T11:20:31Z Normal ExperimentCreated Created Experiment 'alb-rollout-bbf848ddb-2-0' 2024-11-27T11:20:32Z Normal TrafficWeightUpdated Traffic weight updated additional: [{20 } {20 }] time="2024-11-27T11:20:41Z" level=info msg="Deleting e2e-test-name=TestALBExperimentStepNoSetWeight" --- FAIL: TestAWSSuite/TestALBExperimentStepNoSetWeight (21.39s) === FAIL: test/e2e TestAWSSuite (174.69s) === FAIL: test/e2e TestCanarySuite/TestCanaryScaleDownOnAbort (8.12s) time="2024-11-27T11:28:04Z" level=info msg="Deleting e2e-test-name=TestCanaryScaleDownOnAbort" time="2024-11-27T11:28:05Z" level=info msg="service/canary-scaledowndelay-root created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:06Z" level=info msg="service/canary-scaledowndelay-canary created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:07Z" level=info msg="service/canary-scaledowndelay-stable created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:08Z" level=info msg="Warning: annotation \"kubernetes.io/ingress.class\" is deprecated, please use 'spec.ingressClassName' instead\ningress.networking.k8s.io/canary-scaledowndelay-ingress created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:09Z" level=info msg="rollout.argoproj.io/canary-scaledownd-on-abort created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:09Z" level=info msg="Waiting for condition: status=Healthy" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:10Z" level=info msg="Condition 'status=Healthy' met after 1s" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:10Z" level=info msg="Updated rollout pod spec: 2024-11-27T11:28:10.646641521Z" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:10Z" level=info msg="Waiting for condition: status=Paused" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:11Z" level=info msg="Condition 'status=Paused' met after 0s" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:11Z" level=info msg="Aborted rollout" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:11Z" level=info msg="Waiting for condition: status=Degraded" rollout=canary-scaledownd-on-abort time="2024-11-27T11:28:11Z" level=info msg="Condition 'status=Degraded' met after 0s" rollout=canary-scaledownd-on-abort canary_test.go:544: Expected canary-scaledowndelay-canary selector: map[app:canary-scaledowndelay rollouts-pod-template-hash:66597877b7]. Actual: map[app:canary-scaledowndelay rollouts-pod-template-hash:6f47467c87] Name: canary-scaledownd-on-abort Namespace: argo-rollouts Status: ✖ Degraded Message: RolloutAborted: Rollout aborted update to revision 2 Strategy: Canary Step: 0/2 SetWeight: 0 ActualWeight: 0 Images: nginx:1.19-alpine (canary, stable) Replicas: Desired: 1 Current: 2 Updated: 1 Ready: 2 Available: 2 NAME KIND STATUS AGE INFO ⟳ canary-scaledownd-on-abort Rollout ✖ Degraded 2s ├──# revision:2 │ └──⧉ canary-scaledownd-on-abort-6f47467c87 ReplicaSet ✔ Healthy 1s canary,delay:0s │ └──□ canary-scaledownd-on-abort-6f47467c87-4vn5m Pod ✔ Running 1s ready:1/1 └──# revision:1 └──⧉ canary-scaledownd-on-abort-66597877b7 ReplicaSet ✔ Healthy 2s stable └──□ canary-scaledownd-on-abort-66597877b7-7t4ns Pod ✔ Running 2s ready:1/1 --- apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: annotations: rollout.argoproj.io/revision: "2" creationTimestamp: "2024-11-27T11:28:09Z" generation: 3 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestCanaryScaleDownOnAbort name: canary-scaledownd-on-abort namespace: argo-rollouts resourceVersion: "609836" uid: 1e3ee3bd-9aec-4268-914b-a8a78bbabbd8 spec: replicas: 1 selector: matchLabels: app: canary-scaledowndelay strategy: canary: abortScaleDownDelaySeconds: 1 canaryService: canary-scaledowndelay-canary scaleDownDelayRevisionLimit: 1 stableService: canary-scaledowndelay-stable steps: - setCanaryScale: replicas: 1 - pause: {} trafficRouting: alb: ingress: canary-scaledowndelay-ingress rootService: canary-scaledowndelay-root servicePort: 80 template: metadata: annotations: update: "2024-11-27T11:28:10.646641521Z" creationTimestamp: null labels: app: canary-scaledowndelay spec: containers: - image: nginx:1.19-alpine name: canary-scaledowndelay ports: - containerPort: 80 name: http protocol: TCP resources: requests: cpu: 5m memory: 16Mi status: HPAReplicas: 2 abort: true abortedAt: "2024-11-27T11:28:11Z" availableReplicas: 2 blueGreen: {} canary: weights: canary: podTemplateHash: 6f47467c87 serviceName: canary-scaledowndelay-canary weight: 0 stable: podTemplateHash: 66597877b7 serviceName: canary-scaledowndelay-stable weight: 100 conditions: - lastTransitionTime: "2024-11-27T11:28:10Z" lastUpdateTime: "2024-11-27T11:28:10Z" message: Rollout has minimum availability reason: AvailableReason status: "True" type: Available - lastTransitionTime: "2024-11-27T11:28:10Z" lastUpdateTime: "2024-11-27T11:28:10Z" message: Rollout is not healthy reason: RolloutHealthy status: "False" type: Healthy - lastTransitionTime: "2024-11-27T11:28:10Z" lastUpdateTime: "2024-11-27T11:28:10Z" message: RolloutCompleted reason: RolloutCompleted status: "False" type: Completed - lastTransitionTime: "2024-11-27T11:28:11Z" lastUpdateTime: "2024-11-27T11:28:11Z" message: Rollout is paused reason: RolloutPaused status: "True" type: Paused - lastTransitionTime: "2024-11-27T11:28:11Z" lastUpdateTime: "2024-11-27T11:28:11Z" message: Rollout aborted update to revision 2 reason: RolloutAborted status: "False" type: Progressing currentPodHash: 6f47467c87 currentStepHash: 5db8f9b897 currentStepIndex: 0 message: 'RolloutAborted: Rollout aborted update to revision 2' observedGeneration: "3" phase: Degraded readyReplicas: 2 replicas: 2 selector: app=canary-scaledowndelay stableRS: 66597877b7 updatedReplicas: 1 2024-11-27T11:28:09Z Normal RolloutAddedToInformer Rollout resource added to informer: argo-rollouts/canary-scaledownd-on-abort 2024-11-27T11:28:09Z Normal RolloutNotCompleted Rollout not completed, started update to revision 1 (66597877b7) 2024-11-27T11:28:09Z Normal RolloutUpdated Rollout updated to revision 1 2024-11-27T11:28:09Z Normal NewReplicaSetCreated Created ReplicaSet canary-scaledownd-on-abort-66597877b7 (revision 1) 2024-11-27T11:28:09Z Normal PatchingALBIngress Updating Ingress `canary-scaledowndelay-ingress` to desiredWeight '0' 2024-11-27T11:28:09Z Normal TrafficWeightUpdated Traffic weight updated to 0 2024-11-27T11:28:09Z Normal ScalingReplicaSet Scaled up ReplicaSet canary-scaledownd-on-abort-66597877b7 (revision 1) from 0 to 1 2024-11-27T11:28:09Z Normal RolloutCompleted Rollout completed update to revision 1 (66597877b7): Initial deploy 2024-11-27T11:28:10Z (x2) Normal TrafficWeightUpdated Traffic weight updated 2024-11-27T11:28:10Z Normal SwitchService Switched selector for service 'canary-scaledowndelay-stable' from '' to '66597877b7' 2024-11-27T11:28:10Z Normal SwitchService Switched selector for service 'canary-scaledowndelay-canary' from '' to '66597877b7' 2024-11-27T11:28:10Z Normal RolloutNotCompleted Rollout not completed, started update to revision 2 (6f47467c87) 2024-11-27T11:28:10Z Normal RolloutUpdated Rollout updated to revision 2 2024-11-27T11:28:10Z Normal NewReplicaSetCreated Created ReplicaSet canary-scaledownd-on-abort-6f47467c87 (revision 2) 2024-11-27T11:28:10Z Normal ScalingReplicaSet Scaled up ReplicaSet canary-scaledownd-on-abort-6f47467c87 (revision 2) from 0 to 1 2024-11-27T11:28:11Z (x3) Normal SwitchService Switched selector for service 'canary-scaledowndelay-canary' from '66597877b7' to '6f47467c87' 2024-11-27T11:28:11Z Normal RolloutStepCompleted Rollout step 1/2 completed (setCanaryScale{replicas: 1}) 2024-11-27T11:28:11Z Normal RolloutPaused Rollout is paused (CanaryPauseStep) 2024-11-27T11:28:11Z (x3) Normal SwitchService Switched selector for service 'canary-scaledowndelay-canary' from '6f47467c87' to '66597877b7' 2024-11-27T11:28:11Z Warning RolloutAborted Rollout aborted update to revision 2 time="2024-11-27T11:28:11Z" level=info msg="Deleting e2e-test-name=TestCanaryScaleDownOnAbort" --- FAIL: TestCanarySuite/TestCanaryScaleDownOnAbort (8.12s) === FAIL: test/e2e TestCanarySuite (368.30s) time="2024-11-27T11:24:57Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" === FAIL: test/e2e TestExperimentSuite/TestExperimentWithDryRunMetrics (36.76s) time="2024-11-27T11:31:16Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:31:18Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-11-27T11:31:19Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-11-27T11:31:19Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-11-27T11:31:50Z" level=info msg="Condition 'phase=Successful' met after 30s" time="2024-11-27T11:31:50Z" level=info msg="Sleeping 3s" time="2024-11-27T11:31:53Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-11-27T11:31:53Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-11-27T11:31:53Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 34s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 34s delay:26s │ └──□ experiment-with-dry-run-baseline-hx7fs Pod ✔ Running 34s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ 8b5825f3-7def-4baf-a1c9-dc27d4d283ef.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-11-27T11:31:19Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "614161" uid: c009d82c-b28d-4b7c-87e8-33fd16ea091c spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-11-27T11:31:20Z" conditions: - lastTransitionTime: "2024-11-27T11:31:50Z" lastUpdateTime: "2024-11-27T11:31:50Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-11-27T11:31:50Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-11-27T11:31:19Z (x2) Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-11-27T11:31:19Z (x2) Normal ExperimentPending Experiment transitioned from -> Pending 2024-11-27T11:31:19Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-11-27T11:31:20Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-11-27T11:31:20Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-11-27T11:31:20Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-11-27T11:31:20Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-11-27T11:31:50Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-11-27T11:31:50Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-11-27T11:31:50Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-11-27T11:31:53Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (36.76s) === FAIL: test/e2e TestExperimentSuite (128.26s) === FAIL: test/e2e TestFunctionalSuite/TestControllerMetrics (0.66s) time="2024-11-27T11:35:17Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-11-27T11:35:17Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 6133 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc001c8c9c0, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc000d83c00) /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc000b16680?, 0xc0006733c0?, 0xc0017d8660?}, {0x3212d41, 0x4}, {0xc001503e70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc000b16680?, 0xc0006733c0?, 0xc000d83c00?}, {0xc001503e70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc001c8c9c0) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc001c8c9c0, 0xc00112b4d0) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 4875 /usr/local/go/src/testing/testing.go:1648 +0x3ad --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.66s) === FAIL: test/e2e TestFunctionalSuite (404.63s) time="2024-11-27T11:33:37Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" === FAIL: test/e2e TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 1) (8.39s) time="2024-11-27T11:44:27Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:44:29Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-11-27T11:44:31Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-11-27T11:44:33Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route configured\n" rollout=rollout-apisix-canary time="2024-11-27T11:44:35Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:44:35Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-11-27T11:44:35Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:44:36Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (8.39s) === FAIL: test/e2e TestAPISIXSuite (re-run 1) (8.77s) === FAIL: test/e2e TestCanarySuite/TestCanaryScaleDownOnAbort (re-run 1) (10.72s) time="2024-11-27T11:45:22Z" level=info msg="Deleting e2e-test-name=TestCanaryScaleDownOnAbort" time="2024-11-27T11:45:23Z" level=info msg="service/canary-scaledowndelay-root created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:25Z" level=info msg="service/canary-scaledowndelay-canary created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:27Z" level=info msg="service/canary-scaledowndelay-stable created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:28Z" level=info msg="Warning: annotation \"kubernetes.io/ingress.class\" is deprecated, please use 'spec.ingressClassName' instead\ningress.networking.k8s.io/canary-scaledowndelay-ingress created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:30Z" level=info msg="rollout.argoproj.io/canary-scaledownd-on-abort created\n" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:30Z" level=info msg="Waiting for condition: status=Healthy" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:31Z" level=info msg="Condition 'status=Healthy' met after 0s" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:31Z" level=info msg="Updated rollout pod spec: 2024-11-27T11:45:31.101663566Z" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:31Z" level=info msg="Waiting for condition: status=Paused" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:32Z" level=info msg="Condition 'status=Paused' met after 0s" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:32Z" level=info msg="Aborted rollout" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:32Z" level=info msg="Waiting for condition: status=Degraded" rollout=canary-scaledownd-on-abort time="2024-11-27T11:45:32Z" level=info msg="Condition 'status=Degraded' met after 0s" rollout=canary-scaledownd-on-abort canary_test.go:544: Expected canary-scaledowndelay-canary selector: map[app:canary-scaledowndelay rollouts-pod-template-hash:66597877b7]. Actual: map[app:canary-scaledowndelay rollouts-pod-template-hash:f87547876] Name: canary-scaledownd-on-abort Namespace: argo-rollouts Status: ✖ Degraded Message: RolloutAborted: Rollout aborted update to revision 2 Strategy: Canary Step: 0/2 SetWeight: 0 ActualWeight: 0 Images: nginx:1.19-alpine (canary, stable) Replicas: Desired: 1 Current: 2 Updated: 1 Ready: 2 Available: 2 NAME KIND STATUS AGE INFO ⟳ canary-scaledownd-on-abort Rollout ✖ Degraded 2s ├──# revision:2 │ └──⧉ canary-scaledownd-on-abort-f87547876 ReplicaSet ✔ Healthy 1s canary,delay:0s │ └──□ canary-scaledownd-on-abort-f87547876-fr7dj Pod ✔ Running 1s ready:1/1 └──# revision:1 └──⧉ canary-scaledownd-on-abort-66597877b7 ReplicaSet ✔ Healthy 2s stable └──□ canary-scaledownd-on-abort-66597877b7-qj7jq Pod ✔ Running 2s ready:1/1 --- apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: annotations: rollout.argoproj.io/revision: "2" creationTimestamp: "2024-11-27T11:45:30Z" generation: 3 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestCanaryScaleDownOnAbort name: canary-scaledownd-on-abort namespace: argo-rollouts resourceVersion: "627668" uid: 124c499d-86e4-467e-ac90-e7edc70fa588 spec: replicas: 1 selector: matchLabels: app: canary-scaledowndelay strategy: canary: abortScaleDownDelaySeconds: 1 canaryService: canary-scaledowndelay-canary scaleDownDelayRevisionLimit: 1 stableService: canary-scaledowndelay-stable steps: - setCanaryScale: replicas: 1 - pause: {} trafficRouting: alb: ingress: canary-scaledowndelay-ingress rootService: canary-scaledowndelay-root servicePort: 80 template: metadata: annotations: update: "2024-11-27T11:45:31.101663566Z" creationTimestamp: null labels: app: canary-scaledowndelay spec: containers: - image: nginx:1.19-alpine name: canary-scaledowndelay ports: - containerPort: 80 name: http protocol: TCP resources: requests: cpu: 5m memory: 16Mi status: HPAReplicas: 2 abort: true abortedAt: "2024-11-27T11:45:32Z" availableReplicas: 2 blueGreen: {} canary: weights: canary: podTemplateHash: f87547876 serviceName: canary-scaledowndelay-canary weight: 0 stable: podTemplateHash: 66597877b7 serviceName: canary-scaledowndelay-stable weight: 100 conditions: - lastTransitionTime: "2024-11-27T11:45:31Z" lastUpdateTime: "2024-11-27T11:45:31Z" message: Rollout has minimum availability reason: AvailableReason status: "True" type: Available - lastTransitionTime: "2024-11-27T11:45:31Z" lastUpdateTime: "2024-11-27T11:45:31Z" message: Rollout is not healthy reason: RolloutHealthy status: "False" type: Healthy - lastTransitionTime: "2024-11-27T11:45:31Z" lastUpdateTime: "2024-11-27T11:45:31Z" message: RolloutCompleted reason: RolloutCompleted status: "False" type: Completed - lastTransitionTime: "2024-11-27T11:45:32Z" lastUpdateTime: "2024-11-27T11:45:32Z" message: Rollout is paused reason: RolloutPaused status: "True" type: Paused - lastTransitionTime: "2024-11-27T11:45:32Z" lastUpdateTime: "2024-11-27T11:45:32Z" message: Rollout aborted update to revision 2 reason: RolloutAborted status: "False" type: Progressing currentPodHash: f87547876 currentStepHash: 5db8f9b897 currentStepIndex: 0 message: 'RolloutAborted: Rollout aborted update to revision 2' observedGeneration: "3" phase: Degraded readyReplicas: 2 replicas: 2 selector: app=canary-scaledowndelay stableRS: 66597877b7 updatedReplicas: 1 2024-11-27T11:45:30Z Normal RolloutAddedToInformer Rollout resource added to informer: argo-rollouts/canary-scaledownd-on-abort 2024-11-27T11:45:30Z Normal RolloutNotCompleted Rollout not completed, started update to revision 1 (66597877b7) 2024-11-27T11:45:30Z Normal RolloutUpdated Rollout updated to revision 1 2024-11-27T11:45:30Z Normal NewReplicaSetCreated Created ReplicaSet canary-scaledownd-on-abort-66597877b7 (revision 1) 2024-11-27T11:45:30Z Normal PatchingALBIngress Updating Ingress `canary-scaledowndelay-ingress` to desiredWeight '0' 2024-11-27T11:45:30Z Normal TrafficWeightUpdated Traffic weight updated to 0 2024-11-27T11:45:30Z Normal ScalingReplicaSet Scaled up ReplicaSet canary-scaledownd-on-abort-66597877b7 (revision 1) from 0 to 1 2024-11-27T11:45:30Z Normal RolloutCompleted Rollout completed update to revision 1 (66597877b7): Initial deploy 2024-11-27T11:45:31Z (x2) Normal TrafficWeightUpdated Traffic weight updated 2024-11-27T11:45:31Z Normal SwitchService Switched selector for service 'canary-scaledowndelay-stable' from '' to '66597877b7' 2024-11-27T11:45:31Z Normal SwitchService Switched selector for service 'canary-scaledowndelay-canary' from '' to '66597877b7' 2024-11-27T11:45:31Z Normal RolloutNotCompleted Rollout not completed, started update to revision 2 (f87547876) 2024-11-27T11:45:31Z Normal RolloutUpdated Rollout updated to revision 2 2024-11-27T11:45:31Z Normal NewReplicaSetCreated Created ReplicaSet canary-scaledownd-on-abort-f87547876 (revision 2) 2024-11-27T11:45:31Z Normal ScalingReplicaSet Scaled up ReplicaSet canary-scaledownd-on-abort-f87547876 (revision 2) from 0 to 1 2024-11-27T11:45:32Z (x3) Normal SwitchService Switched selector for service 'canary-scaledowndelay-canary' from '66597877b7' to 'f87547876' 2024-11-27T11:45:32Z Normal RolloutStepCompleted Rollout step 1/2 completed (setCanaryScale{replicas: 1}) 2024-11-27T11:45:32Z Normal RolloutPaused Rollout is paused (CanaryPauseStep) 2024-11-27T11:45:32Z (x3) Normal SwitchService Switched selector for service 'canary-scaledowndelay-canary' from 'f87547876' to '66597877b7' 2024-11-27T11:45:32Z Warning RolloutAborted Rollout aborted update to revision 2 time="2024-11-27T11:45:32Z" level=info msg="Deleting e2e-test-name=TestCanaryScaleDownOnAbort" time="2024-11-27T11:45:32Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestCanarySuite/TestCanaryScaleDownOnAbort (10.72s) === FAIL: test/e2e TestCanarySuite (re-run 1) (12.93s) time="2024-11-27T11:45:22Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" === FAIL: test/e2e TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 1) (38.63s) time="2024-11-27T11:45:42Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:45:44Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-11-27T11:45:46Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-11-27T11:45:46Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-11-27T11:46:18Z" level=info msg="Condition 'phase=Successful' met after 31s" time="2024-11-27T11:46:18Z" level=info msg="Sleeping 3s" time="2024-11-27T11:46:21Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-11-27T11:46:21Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-11-27T11:46:21Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 35s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 35s delay:26s │ └──□ experiment-with-dry-run-baseline-jspbf Pod ✔ Running 35s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ 05828cdb-f0b1-4bdb-8c1c-b989222baf50.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-11-27T11:45:46Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "628184" uid: c417c693-29ab-4e9d-ba7b-d0e858fd4b95 spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-11-27T11:45:48Z" conditions: - lastTransitionTime: "2024-11-27T11:46:18Z" lastUpdateTime: "2024-11-27T11:46:18Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-11-27T11:46:18Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-11-27T11:45:46Z (x2) Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-11-27T11:45:46Z (x2) Normal ExperimentPending Experiment transitioned from -> Pending 2024-11-27T11:45:46Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-11-27T11:45:48Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-11-27T11:45:48Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-11-27T11:45:48Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-11-27T11:45:48Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-11-27T11:46:18Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-11-27T11:46:18Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-11-27T11:46:18Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-11-27T11:46:21Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:46:21Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (38.63s) === FAIL: test/e2e TestExperimentSuite (re-run 1) (38.98s) === FAIL: test/e2e TestFunctionalSuite/TestControllerMetrics (re-run 1) (0.54s) time="2024-11-27T11:46:33Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-11-27T11:46:33Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 95 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc0005009c0, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc0007bfa00) /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc00080ed40?, 0xc000a06f40?, 0xc000308320?}, {0x3212d41, 0x4}, {0xc000c1de70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc00080ed40?, 0xc000a06f40?, 0xc0007bfa00?}, {0xc000c1de70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc0005009c0) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc0005009c0, 0xc000c2ba70) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 82 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-11-27T11:46:34Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.54s) === FAIL: test/e2e TestFunctionalSuite (re-run 1) (2.76s) time="2024-11-27T11:46:33Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" === FAIL: test/e2e TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 2) (8.69s) time="2024-11-27T11:46:44Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:46:46Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-11-27T11:46:48Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-11-27T11:46:50Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:46:52Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:46:52Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-11-27T11:46:52Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:46:53Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (8.69s) === FAIL: test/e2e TestAPISIXSuite (re-run 2) (9.04s) === FAIL: test/e2e TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 2) (38.36s) time="2024-11-27T11:47:30Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:47:32Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-11-27T11:47:34Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-11-27T11:47:34Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-11-27T11:48:05Z" level=info msg="Condition 'phase=Successful' met after 30s" time="2024-11-27T11:48:05Z" level=info msg="Sleeping 3s" time="2024-11-27T11:48:08Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-11-27T11:48:08Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-11-27T11:48:08Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 34s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 34s delay:26s │ └──□ experiment-with-dry-run-baseline-nmhcq Pod ✔ Running 34s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ 37841be3-5dab-4086-aae9-d6d925618b61.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-11-27T11:47:34Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "629336" uid: ec99d88d-3ee7-4561-9573-78d054e953d7 spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-11-27T11:47:35Z" conditions: - lastTransitionTime: "2024-11-27T11:48:05Z" lastUpdateTime: "2024-11-27T11:48:05Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-11-27T11:48:05Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-11-27T11:47:34Z Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-11-27T11:47:34Z Normal ExperimentPending Experiment transitioned from -> Pending 2024-11-27T11:47:34Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-11-27T11:47:35Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-11-27T11:47:35Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-11-27T11:47:35Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-11-27T11:47:35Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-11-27T11:48:05Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-11-27T11:48:05Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-11-27T11:48:05Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-11-27T11:48:08Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:48:08Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (38.36s) === FAIL: test/e2e TestExperimentSuite (re-run 2) (38.76s) === FAIL: test/e2e TestFunctionalSuite/TestControllerMetrics (re-run 2) (0.52s) time="2024-11-27T11:48:20Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-11-27T11:48:20Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 46 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc000103520, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc00052a800) /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc0003136c0?, 0xc0009361d8?, 0xc0005543a0?}, {0x3212d41, 0x4}, {0xc000819e70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc0003136c0?, 0xc0009361d8?, 0xc00052a800?}, {0xc000819e70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc000103520) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc000103520, 0xc000910f30) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 71 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-11-27T11:48:20Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.52s) === FAIL: test/e2e TestFunctionalSuite (re-run 2) (2.50s) time="2024-11-27T11:48:20Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" === FAIL: test/e2e TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 3) (7.50s) time="2024-11-27T11:48:30Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:48:32Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-11-27T11:48:34Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-11-27T11:48:36Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:48:37Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:48:37Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-11-27T11:48:37Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:48:38Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (7.50s) === FAIL: test/e2e TestAPISIXSuite (re-run 3) (7.86s) === FAIL: test/e2e TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 3) (37.98s) time="2024-11-27T11:48:48Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:48:50Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-11-27T11:48:52Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-11-27T11:48:52Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-11-27T11:49:23Z" level=info msg="Condition 'phase=Successful' met after 30s" time="2024-11-27T11:49:23Z" level=info msg="Sleeping 3s" time="2024-11-27T11:49:26Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-11-27T11:49:26Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-11-27T11:49:26Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 34s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 34s delay:26s │ └──□ experiment-with-dry-run-baseline-lvlbd Pod ✔ Running 34s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ 71ef3d10-ad82-411d-8084-62ca2d5e76d3.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-11-27T11:48:52Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "629956" uid: 403a5f20-b6ab-4f6c-a5d1-9710712a91d8 spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-11-27T11:48:53Z" conditions: - lastTransitionTime: "2024-11-27T11:49:23Z" lastUpdateTime: "2024-11-27T11:49:23Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-11-27T11:49:23Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-11-27T11:48:52Z Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-11-27T11:48:52Z Normal ExperimentPending Experiment transitioned from -> Pending 2024-11-27T11:48:52Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-11-27T11:48:53Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-11-27T11:48:53Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-11-27T11:48:53Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-11-27T11:48:53Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-11-27T11:49:23Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-11-27T11:49:23Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-11-27T11:49:23Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-11-27T11:49:26Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:49:26Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (37.98s) === FAIL: test/e2e TestExperimentSuite (re-run 3) (38.40s) === FAIL: test/e2e TestFunctionalSuite/TestControllerMetrics (re-run 3) (0.54s) time="2024-11-27T11:49:38Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-11-27T11:49:38Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 95 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc000c824e0, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc00050db00) /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc000d2cac0?, 0xc000d0d378?, 0xc000122160?}, {0x3212d41, 0x4}, {0xc0003bfe70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc000d2cac0?, 0xc000d0d378?, 0xc00050db00?}, {0xc0003bfe70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc000c824e0) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc000c824e0, 0xc0002f5050) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 28 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-11-27T11:49:39Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.54s) === FAIL: test/e2e TestFunctionalSuite (re-run 3) (2.74s) time="2024-11-27T11:49:38Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" === FAIL: test/e2e TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 4) (7.53s) time="2024-11-27T11:49:49Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:49:51Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-11-27T11:49:53Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-11-27T11:49:54Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:49:56Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:49:56Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-11-27T11:49:56Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:49:56Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (7.53s) === FAIL: test/e2e TestAPISIXSuite (re-run 4) (7.87s) === FAIL: test/e2e TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 4) (37.90s) time="2024-11-27T11:50:06Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:50:08Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-11-27T11:50:10Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-11-27T11:50:10Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-11-27T11:50:41Z" level=info msg="Condition 'phase=Successful' met after 30s" time="2024-11-27T11:50:41Z" level=info msg="Sleeping 3s" time="2024-11-27T11:50:44Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-11-27T11:50:44Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-11-27T11:50:44Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 34s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 34s delay:26s │ └──□ experiment-with-dry-run-baseline-h2qq5 Pod ✔ Running 34s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ d1e3a87a-7f20-47b8-b2ff-035c046de2d9.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-11-27T11:50:10Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "630574" uid: 43d916ec-745f-4c99-811d-37e90729f98a spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-11-27T11:50:11Z" conditions: - lastTransitionTime: "2024-11-27T11:50:41Z" lastUpdateTime: "2024-11-27T11:50:41Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-11-27T11:50:41Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-11-27T11:50:10Z (x2) Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-11-27T11:50:10Z (x2) Normal ExperimentPending Experiment transitioned from -> Pending 2024-11-27T11:50:10Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-11-27T11:50:11Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-11-27T11:50:11Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-11-27T11:50:11Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-11-27T11:50:11Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-11-27T11:50:41Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-11-27T11:50:41Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-11-27T11:50:41Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-11-27T11:50:44Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:50:44Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (37.90s) === FAIL: test/e2e TestExperimentSuite (re-run 4) (38.22s) === FAIL: test/e2e TestFunctionalSuite/TestControllerMetrics (re-run 4) (0.57s) time="2024-11-27T11:50:57Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-11-27T11:50:57Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 40 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc000b6e000, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc0006eb000) /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc0004fd300?, 0xc000131638?, 0xc000b06fe0?}, {0x3212d41, 0x4}, {0xc00086be70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc0004fd300?, 0xc000131638?, 0xc0006eb000?}, {0xc00086be70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc000b6e000) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc000b6e000, 0xc000b1a5a0) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 82 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-11-27T11:50:58Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.57s) === FAIL: test/e2e TestFunctionalSuite (re-run 4) (3.65s) time="2024-11-27T11:50:57Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" === FAIL: test/e2e TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 5) (11.43s) time="2024-11-27T11:51:08Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:51:12Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-11-27T11:51:14Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-11-27T11:51:16Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:51:18Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-11-27T11:51:18Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-11-27T11:51:18Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-11-27T11:51:20Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (11.43s) === FAIL: test/e2e TestAPISIXSuite (re-run 5) (14.51s) === FAIL: test/e2e TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 5) (47.79s) time="2024-11-27T11:51:32Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:51:38Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-11-27T11:51:46Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-11-27T11:51:46Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-11-27T11:52:17Z" level=info msg="Condition 'phase=Successful' met after 30s" time="2024-11-27T11:52:17Z" level=info msg="Sleeping 3s" time="2024-11-27T11:52:20Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-11-27T11:52:20Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-11-27T11:52:20Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 34s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 34s delay:26s │ └──□ experiment-with-dry-run-baseline-4nxmx Pod ✔ Running 34s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ cc10b951-1dcd-4a60-94a7-1b8bb0b154b2.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-11-27T11:51:46Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "631410" uid: 98aa4cfb-165f-4c93-8158-9ccddf6d4109 spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-11-27T11:51:47Z" conditions: - lastTransitionTime: "2024-11-27T11:52:17Z" lastUpdateTime: "2024-11-27T11:52:17Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-11-27T11:52:17Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-11-27T11:51:46Z (x2) Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-11-27T11:51:46Z (x2) Normal ExperimentPending Experiment transitioned from -> Pending 2024-11-27T11:51:46Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-11-27T11:51:47Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-11-27T11:51:47Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-11-27T11:51:47Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-11-27T11:51:47Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-11-27T11:52:17Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-11-27T11:52:17Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-11-27T11:52:17Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-11-27T11:52:20Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-11-27T11:52:20Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (47.79s) === FAIL: test/e2e TestExperimentSuite (re-run 5) (48.14s) === FAIL: test/e2e TestFunctionalSuite/TestControllerMetrics (re-run 5) (0.61s) time="2024-11-27T11:52:33Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-11-27T11:52:33Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 89 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc000602d00, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc0001b3100) /tmp/tmp.aCMO1twgu4/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc000810840?, 0xc00061e1b0?, 0xc000c903d0?}, {0x3212d41, 0x4}, {0xc000c03e70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc000810840?, 0xc00061e1b0?, 0xc0001b3100?}, {0xc000c03e70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc000602d00) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc000602d00, 0xc000b1e5a0) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 82 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-11-27T11:52:33Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.61s) === FAIL: test/e2e TestFunctionalSuite (re-run 5) (3.49s) time="2024-11-27T11:52:33Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" DONE 6 runs, 140 tests, 6 skipped, 42 failures in 2947.056s make: *** [Makefile:239: test-e2e] Error 1 + set +x ----------------------------------------------------------------- These were the tests that succeeded: PASS test/e2e PASS test/e2e PASS test/e2e.TestAPISIXSuite/TestAPISIXCanaryStep (28.17s) PASS test/e2e.TestAWSSuite (re-run 1) (24.30s) PASS test/e2e.TestAWSSuite/TestALBExperimentStep (23.73s) PASS test/e2e.TestAWSSuite/TestALBExperimentStepMultiIngress (24.17s) PASS test/e2e.TestAWSSuite/TestALBExperimentStepNoSetWeight (re-run 1) (23.92s) PASS test/e2e.TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress (25.82s) PASS test/e2e.TestAWSSuite/TestALBPingPongUpdate (17.98s) PASS test/e2e.TestAWSSuite/TestALBPingPongUpdateMultiIngress (17.98s) PASS test/e2e.TestAWSSuite/TestAlbHeaderRoute (14.40s) PASS test/e2e.TestAWSSuite/TestAlbHeaderRouteMultiIngress (27.08s) PASS test/e2e.TestAnalysisSuite (319.32s) PASS test/e2e.TestAnalysisSuite/TestAnalysisWithArgs (39.63s) PASS test/e2e.TestAnalysisSuite/TestAnalysisWithSecret (29.99s) PASS test/e2e.TestAnalysisSuite/TestBackgroundAnalysisWithArgs (29.39s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenAbortAndUpdate (22.29s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenAnalysis (33.26s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenKitchenSink (47.03s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenPostPromotionFail (17.42s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenPrePromotionFail (28.40s) PASS test/e2e.TestAnalysisSuite/TestCanaryBackgroundAnalysis (4.72s) PASS test/e2e.TestAnalysisSuite/TestCanaryFailInlineMultipleAnalysis (12.03s) PASS test/e2e.TestAnalysisSuite/TestCanaryInlineAnalysis (18.09s) PASS test/e2e.TestAnalysisSuite/TestCanaryInlineMultipleAnalysis (17.33s) PASS test/e2e.TestAnalysisSuite/TestMultipleAnalysis (10.63s) PASS test/e2e.TestBlueGreenSuite (150.35s) PASS test/e2e.TestBlueGreenSuite/TestBlueGreenPreviewReplicaCount (8.45s) PASS test/e2e.TestBlueGreenSuite/TestBlueGreenProgressDeadlineExceededWithPause (64.93s) PASS test/e2e.TestBlueGreenSuite/TestBlueGreenProgressDeadlineExceededWithoutPause (65.42s) PASS test/e2e.TestBlueGreenSuite/TestEphemeralMetadata (11.21s) PASS test/e2e.TestCanarySuite (re-run 2) (16.94s) PASS test/e2e.TestCanarySuite/TestCanaryDynamicStableScale (27.20s) PASS test/e2e.TestCanarySuite/TestCanaryDynamicStableScaleRollbackToStable (21.18s) PASS test/e2e.TestCanarySuite/TestCanaryProgressDeadlineExceededWithPause (124.53s) PASS test/e2e.TestCanarySuite/TestCanaryScaleDownDelay (13.59s) PASS test/e2e.TestCanarySuite/TestCanaryScaleDownOnAbort (re-run 2) (14.67s) PASS test/e2e.TestCanarySuite/TestCanaryScaleDownOnAbortNoTrafficRouting (10.90s) PASS test/e2e.TestCanarySuite/TestCanarySetCanaryScale (12.75s) PASS test/e2e.TestCanarySuite/TestCanaryUnScaleDownOnAbort (10.74s) PASS test/e2e.TestCanarySuite/TestCanaryWithPausedRollout (101.70s) PASS test/e2e.TestCanarySuite/TestEphemeralMetadata (7.58s) PASS test/e2e.TestCanarySuite/TestReduceWeightAndHonorMaxUnavailable (28.20s) PASS test/e2e.TestCanarySuite/TestRolloutScalingDuringUpdate (3.19s) PASS test/e2e.TestCanarySuite/TestRolloutScalingWhenPaused (5.78s) PASS test/e2e.TestCanarySuite/TestRolloutWithMaxSurgeScalingDuringUpdate (3.29s) PASS test/e2e.TestExperimentSuite/TestExperimentWithMeasurementRetentionMetrics (36.80s) PASS test/e2e.TestExperimentSuite/TestExperimentWithMultiportServiceAndScaleDownDelay (18.06s) PASS test/e2e.TestExperimentSuite/TestExperimentWithServiceAndScaleDownDelay (18.03s) PASS test/e2e.TestExperimentSuite/TestExperimentWithServiceNameAndScaleDownDelay (17.97s) PASS test/e2e.TestExperimentSuite/TestRolloutWithExperimentAndAnalysis (10.25s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenExceedProgressDeadlineAbort (11.46s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenPreviewReplicaCountPromoteFull (9.55s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenPromoteFull (8.07s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenScaleDownDelay (10.35s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenScaleDownOnAbort (11.94s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenToCanary (6.92s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenUpdate (7.08s) PASS test/e2e.TestFunctionalSuite/TestCanaryPromoteFull (8.91s) PASS test/e2e.TestFunctionalSuite/TestCompleteRolloutRestart (21.82s) PASS test/e2e.TestFunctionalSuite/TestContainerResourceFormats (4.25s) PASS test/e2e.TestFunctionalSuite/TestFixInvalidSpec (22.50s) PASS test/e2e.TestFunctionalSuite/TestKubectlWaitForCompleted (24.63s) PASS test/e2e.TestFunctionalSuite/TestKubectlWaitForPaused (7.75s) PASS test/e2e.TestFunctionalSuite/TestMalformedRollout (3.66s) PASS test/e2e.TestFunctionalSuite/TestMalformedRolloutEphemeralCtr (3.84s) PASS test/e2e.TestFunctionalSuite/TestRolloutAbortRetryPromote (10.04s) PASS test/e2e.TestFunctionalSuite/TestRolloutBadToBadUpdate (23.98s) PASS test/e2e.TestFunctionalSuite/TestRolloutBadToGoodSetWeightUpdate (22.39s) PASS test/e2e.TestFunctionalSuite/TestRolloutBadToGoodUpdate (17.07s) PASS test/e2e.TestFunctionalSuite/TestRolloutGoodToBadUpdate (25.50s) PASS test/e2e.TestFunctionalSuite/TestRolloutPDBRestart (48.78s) PASS test/e2e.TestFunctionalSuite/TestRolloutPauseDurationGreaterThanProgressDeadlineSeconds (37.68s) PASS test/e2e.TestFunctionalSuite/TestRolloutPodVolumesItemsSpec (11.35s) PASS test/e2e.TestFunctionalSuite/TestRolloutRestart (18.14s) PASS test/e2e.TestFunctionalSuite/TestWorkloadRef (19.58s) PASS test/e2e.TestFunctionalSuite/TestWorkloadRefTemplate (4.33s) PASS test/e2e.TestHeaderRoutingSuite (19.81s) PASS test/e2e.TestHeaderRoutingSuite/TestIstioHostHeaderRoute (19.48s) PASS test/e2e.TestIstioSuite (133.89s) PASS test/e2e.TestIstioSuite/TestIstioAbortUpdate (11.25s) PASS test/e2e.TestIstioSuite/TestIstioAbortUpdateDeleteAllCanaryPods (11.87s) PASS test/e2e.TestIstioSuite/TestIstioHostSplit (22.62s) PASS test/e2e.TestIstioSuite/TestIstioHostSplitExperimentStep (21.04s) PASS test/e2e.TestIstioSuite/TestIstioHostSplitOnlyTls (11.10s) PASS test/e2e.TestIstioSuite/TestIstioSubsetSplit (13.72s) PASS test/e2e.TestIstioSuite/TestIstioSubsetSplitExperimentStep (20.27s) PASS test/e2e.TestIstioSuite/TestIstioSubsetSplitSingleRoute (10.49s) PASS test/e2e.TestIstioSuite/TestIstioUpdateInMiddleZeroCanaryReplicas (11.19s) PASS test/e2e.TestMirrorRouteSuite (17.57s) PASS test/e2e.TestMirrorRouteSuite/TestIstioHostMirrorRoute (17.23s) PASS test/e2e.TestRollbackSuite (60.34s) PASS test/e2e.TestRollbackSuite/TestRollbackAnalysisOutsideWindow (31.50s) PASS test/e2e.TestRollbackSuite/TestRollbackAnalysisWithinWindow (24.01s) These were the tests that failed: --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (11.43s) --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (7.50s) --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (7.53s) --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (7.90s) --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (8.39s) --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (8.69s) --- FAIL: TestAWSSuite/TestALBExperimentStepNoSetWeight (21.39s) --- FAIL: TestCanarySuite/TestCanaryScaleDownOnAbort (10.72s) --- FAIL: TestCanarySuite/TestCanaryScaleDownOnAbort (8.12s) --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (36.76s) --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (37.90s) --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (37.98s) --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (38.36s) --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (38.63s) --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (47.79s) --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.52s) --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.54s) --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.57s) --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.61s) --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.66s) E2E tests file is complete: DONE 6 runs, 140 tests, 6 skipped, 42 failures in 2947.056s * SUCCESS: No unexpected errors occurred. * Cleaning up hack/run-upstream-argo-rollouts-e2e-tests.sh: line 7: killall: command not found hack/run-upstream-argo-rollouts-e2e-tests.sh: line 8: killall: command not found + [[ 0 != 0 ]] + kubectl delete ns argo-rollouts namespace "argo-rollouts" deleted + kubectl wait --timeout=5m --for=delete namespace/argo-rollouts + kubectl create ns argo-rollouts namespace/argo-rollouts created + kubectl config set-context --current --namespace=argo-rollouts Context "default/api-psi-417-ocp-gitops-qe-com:6443/kube:admin" modified. + cat + kubectl apply -f /tmp/tmp.TxJP1GNlai/rollout-manager.yaml rolloutmanager.argoproj.io/argo-rollout created + cd /tmp/tmp.TxJP1GNlai + git clone https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-openshift Cloning into 'rollouts-plugin-trafficrouter-openshift'... + cd /tmp/tmp.TxJP1GNlai/rollouts-plugin-trafficrouter-openshift + git checkout 8b4125a7f9ecffb0247df91a4c890f88c0c523b7 Note: switching to '8b4125a7f9ecffb0247df91a4c890f88c0c523b7'. You are in 'detached HEAD' state. You can look around, make experimental changes and commit them, and you can discard any commits you make in this state without impacting any branches by switching back to a branch. If you want to create a new branch to retain commits you create, you may do so (now or later) by using -c with the switch command. Example: git switch -c Or undo this operation with: git switch - Turn off this advice by setting config variable advice.detachedHead to false HEAD is now at 8b4125a Switch lint to use Makefile, fix go.mod + make test-e2e go test -v -p=1 -timeout=20m -race -count=1 -coverprofile=coverage.out ./tests/e2e go: downloading go1.22.2 (linux/amd64) go: downloading github.com/onsi/ginkgo/v2 v2.19.0 go: downloading github.com/openshift/api v0.0.0-20230417092139-1b2161d23365 go: downloading github.com/onsi/gomega v1.33.1 go: downloading github.com/argoproj/argo-rollouts v1.6.6 go: downloading k8s.io/client-go v0.26.3 go: downloading k8s.io/apimachinery v0.26.3 go: downloading github.com/openshift/client-go v0.0.0-20230419131419-497c7032c581 go: downloading k8s.io/api v0.26.3 go: downloading github.com/google/go-cmp v0.6.0 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading github.com/go-logr/logr v1.4.1 go: downloading golang.org/x/net v0.25.0 go: downloading k8s.io/klog/v2 v2.110.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading golang.org/x/sys v0.20.0 go: downloading golang.org/x/text v0.15.0 go: downloading k8s.io/cli-runtime v0.26.3 go: downloading k8s.io/kubernetes v1.26.3 go: downloading golang.org/x/term v0.20.0 go: downloading google.golang.org/grpc v1.57.1 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/xlab/treeprint v1.2.0 go: downloading k8s.io/apiserver v0.26.3 go: downloading k8s.io/component-base v0.26.3 go: downloading k8s.io/component-helpers v0.26.3 === RUN TestPlugin Running Suite: Plugin e2e Tests Suite - /tmp/tmp.TxJP1GNlai/rollouts-plugin-trafficrouter-openshift/tests/e2e ============================================================================================================= Random Seed: 1732708711 Will run 3 of 3 specs service/rollouts-demo-stable created service/rollouts-demo-canary created route.route.openshift.io/rollouts-demo created rollout.argoproj.io/rollouts-demo created Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy W1127 11:58:49.579180 78317 warnings.go:70] unknown field "spec.template.metadata.creationTimestamp" Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy •service/rollouts-demo-stable created service/rollouts-demo-canary created route.route.openshift.io/rollouts-demo-prod created route.route.openshift.io/rollouts-demo-stage created rollout.argoproj.io/rollouts-demo created Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy W1127 11:59:21.888259 78317 warnings.go:70] unknown field "spec.template.metadata.creationTimestamp" Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy Rollout phase mismatch, got Paused, want Healthy ------------------------------ • [FAILED] [52.932 seconds] OpenShift Route Traffic Plugin Tests Test the functionality of the Route plugin [It] should handle multiple Routes that are specified in the plugin /tmp/tmp.TxJP1GNlai/rollouts-plugin-trafficrouter-openshift/tests/e2e/plugin_test.go:176 Timeline >> STEP: verify if Rollout is healthy @ 11/27/24 11:59:16.694 STEP: verify if the Routes have the required weights before update @ 11/27/24 11:59:21.812 STEP: update the Rollout and verify if it has reached Paused phase @ 11/27/24 11:59:21.848 STEP: verify if the Routes have been updated with the correct weights @ 11/27/24 11:59:25.974 STEP: promote the Rollout for the final time and verify if the previous version is removed @ 11/27/24 11:59:26.01 [FAILED] in [It] - /tmp/tmp.TxJP1GNlai/rollouts-plugin-trafficrouter-openshift/tests/e2e/plugin_test.go:227 @ 11/27/24 11:59:56.05 << Timeline [FAILED] Timed out after 30.000s. Expected : false to be true In [It] at: /tmp/tmp.TxJP1GNlai/rollouts-plugin-trafficrouter-openshift/tests/e2e/plugin_test.go:227 @ 11/27/24 11:59:56.05 ------------------------------ service/rollouts-demo-stable created service/rollouts-demo-canary created route.route.openshift.io/rollouts-demo created analysistemplate.argoproj.io/sample-template created rollout.argoproj.io/rollouts-demo created Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy W1127 12:00:17.077315 78317 warnings.go:70] unknown field "spec.template.metadata.creationTimestamp" Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy • Summarizing 1 Failure: [FAIL] OpenShift Route Traffic Plugin Tests Test the functionality of the Route plugin [It] should handle multiple Routes that are specified in the plugin /tmp/tmp.TxJP1GNlai/rollouts-plugin-trafficrouter-openshift/tests/e2e/plugin_test.go:227 Ran 3 of 3 Specs in 129.839 seconds FAIL! -- 2 Passed | 1 Failed | 0 Pending | 0 Skipped --- FAIL: TestPlugin (129.84s) FAIL coverage: [no statements] FAIL github.com/argoproj-labs/rollouts-plugin-trafficrouter-openshift/tests/e2e 129.947s FAIL make: *** [Makefile:32: test-e2e] Error 1 + exit_code=2 + [[ 2 != 0 ]] + echo 2 + tee /tekton/results/result 2 + failed=2 + oc patch subscription openshift-gitops-operator -n openshift-operators --type json --patch '[{"op": "remove", "path": "/spec/config"}]' subscription.operators.coreos.com/openshift-gitops-operator patched + update_result + [[ 2 != 0 ]] + echo test-argo-rollouts + tee /install-dir/psi-417-result.txt test-argo-rollouts + exit 0