STEP-TEST-ARGO-ROLLOUTS + RESULT=/install-dir/psi-416-result.txt Logging in as kubeadmin user + trap update_result EXIT + echo -e 'Logging in as kubeadmin user' ++ cat /install-dir/psi-416/auth/kubeadmin-password ++ cat /install-dir/psi-416/auth/api-url + oc login -u kubeadmin -p U6UhY-aU6D3-VdigG-XjbWz https://api.psi-416.ocp-gitops-qe.com:6443 --insecure-skip-tls-verify=true WARNING: Using insecure TLS client config. Setting this option is not supported! Login successful. You have access to 76 projects, the list has been suppressed. You can list all projects with 'oc projects' Using project "default". Welcome! See 'oc help' to get started. ++ echo true ++ tr '[:upper:]' '[:lower:]' + [[ true == \f\a\l\s\e ]] + exit_code=0 + failed=0 ++ oc get deployment openshift-gitops-operator-controller-manager -n openshift-gitops-operator -o 'jsonpath={.metadata.namespace}' --ignore-not-found + openshift_gitops_operator= + operator_namespace=openshift-operators + oc patch subscription openshift-gitops-operator -n openshift-operators --type merge --patch '{"spec": {"config": {"env": [{"name": "CLUSTER_SCOPED_ARGO_ROLLOUTS_NAMESPACES", "value": "argo-rollouts,test-rom-ns-1,rom-ns-1"}]}}}' subscription.operators.coreos.com/openshift-gitops-operator patched ++ mktemp -d + ROLLOUTS_TMP_DIR=/tmp/tmp.mWLvz2au9x + cd /tmp/tmp.mWLvz2au9x + git clone https://github.com/argoproj-labs/argo-rollouts-manager Cloning into 'argo-rollouts-manager'... + cd /tmp/tmp.mWLvz2au9x/argo-rollouts-manager + TARGET_ROLLOUT_MANAGER_COMMIT=e054d35fd0d9dad6b9ae26efa60f43cadc194b4b + TARGET_OPENSHIFT_ROUTE_ROLLOUT_PLUGIN_COMMIT=8b4125a7f9ecffb0247df91a4c890f88c0c523b7 + git checkout e054d35fd0d9dad6b9ae26efa60f43cadc194b4b Note: switching to 'e054d35fd0d9dad6b9ae26efa60f43cadc194b4b'. You are in 'detached HEAD' state. You can look around, make experimental changes and commit them, and you can discard any commits you make in this state without impacting any branches by switching back to a branch. If you want to create a new branch to retain commits you create, you may do so (now or later) by using -c with the switch command. Example: git switch -c Or undo this operation with: git switch - Turn off this advice by setting config variable advice.detachedHead to false HEAD is now at e054d35 Backport cluster/namespace-scoped RBAC tests to 0.0.3 + make test-e2e NAMESPACE_SCOPED_ARGO_ROLLOUTS=false hack/run-rollouts-manager-e2e-tests.sh + '[' false == true ']' + go test -v -p=1 -timeout=30m -race -count=1 -coverprofile=coverage.out ./tests/e2e/cluster-scoped go: downloading sigs.k8s.io/controller-runtime v0.16.3 go: downloading go.uber.org/zap v1.25.0 go: downloading github.com/onsi/ginkgo/v2 v2.11.0 go: downloading k8s.io/apimachinery v0.28.3 go: downloading k8s.io/api v0.28.3 go: downloading k8s.io/client-go v0.28.3 go: downloading github.com/onsi/gomega v1.27.10 go: downloading github.com/coreos/prometheus-operator v0.40.0 go: downloading k8s.io/apiextensions-apiserver v0.28.3 go: downloading github.com/go-logr/logr v1.2.4 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/google/go-cmp v0.5.9 go: downloading github.com/go-logr/zapr v1.2.4 go: downloading github.com/evanphx/json-patch/v5 v5.6.0 go: downloading github.com/evanphx/json-patch v5.6.0+incompatible go: downloading github.com/gogo/protobuf v1.3.2 go: downloading k8s.io/klog/v2 v2.100.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.2.3 go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 go: downloading sigs.k8s.io/yaml v1.3.0 go: downloading golang.org/x/net v0.23.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 go: downloading github.com/pkg/errors v0.9.1 go: downloading golang.org/x/sys v0.18.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading github.com/json-iterator/go v1.1.12 go: downloading gomodules.xyz/jsonpatch/v2 v2.4.0 go: downloading k8s.io/component-base v0.28.3 go: downloading golang.org/x/text v0.14.0 go: downloading golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 go: downloading github.com/prometheus/client_golang v1.16.0 go: downloading github.com/davecgh/go-spew v1.1.1 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/google/gnostic-models v0.6.8 go: downloading github.com/imdario/mergo v0.3.13 go: downloading github.com/spf13/pflag v1.0.5 go: downloading golang.org/x/term v0.18.0 go: downloading github.com/golang/protobuf v1.5.3 go: downloading github.com/google/uuid v1.3.0 go: downloading github.com/fsnotify/fsnotify v1.6.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading golang.org/x/time v0.3.0 go: downloading golang.org/x/oauth2 v0.10.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 go: downloading github.com/go-openapi/swag v0.22.3 go: downloading github.com/go-openapi/jsonreference v0.20.2 go: downloading github.com/prometheus/client_model v0.4.0 go: downloading github.com/prometheus/common v0.44.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.2.0 go: downloading github.com/prometheus/procfs v0.10.1 go: downloading github.com/emicklei/go-restful/v3 v3.11.0 go: downloading github.com/go-openapi/jsonpointer v0.19.6 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.4 go: downloading github.com/josharian/intern v1.0.0 === RUN TestClusterScoped Running Suite: Cluster-scoped Suite - /tmp/tmp.mWLvz2au9x/argo-rollouts-manager/tests/e2e/cluster-scoped ======================================================================================================== Random Seed: 1730296584 Will run 6 of 6 specs HavePhase: expected: Available actual: HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Available HaveSuccessCondition: expected: {Reconciled True 0 0001-01-01 00:00:00 +0000 UTC Success } actual: {Reconciled True 0 2024-10-30 13:56:34 +0000 UTC Success } Object exists in ExistByName: argo-rollouts Object exists in ExistByName: argo-rollouts Object exists in ExistByName: argo-rollouts-aggregate-to-admin Object exists in ExistByName: argo-rollouts-aggregate-to-edit Object exists in ExistByName: argo-rollouts-aggregate-to-view Object exists in ExistByName: argo-rollouts ------------------------------ • [FAILED] [27.212 seconds] Cluster-scoped RolloutManager tests Testing cluster-scoped RolloutManager behaviour [It] After creating cluster-scoped RolloutManager in one namespace, operator should create appropriate K8s resources and watch argo rollouts CR in another namespace. /tmp/tmp.mWLvz2au9x/argo-rollouts-manager/tests/e2e/cluster-scoped/cluster_scoped_rollouts_test.go:49 Timeline >> STEP: Create cluster-scoped RolloutManager in default namespace. @ 10/30/24 13:56:34.5 STEP: Verify that RolloutManager is successfully created. @ 10/30/24 13:56:34.539 STEP: Verify that Status.Condition is having success condition. @ 10/30/24 13:56:51.103 STEP: Verify that expected resources are created. @ 10/30/24 13:56:51.134 STEP: Verify that ServiceAccount is created. @ 10/30/24 13:56:51.134 STEP: Verify that ServiceAccount has correct labels and annotations. @ 10/30/24 13:56:51.162 STEP: Verify that argo-rollout ClusterRoles is created. @ 10/30/24 13:56:51.162 STEP: Verify that ClusterRole has correct labels and annotations. @ 10/30/24 13:56:51.186 STEP: Verify that ClusterRole has correct policy rules. @ 10/30/24 13:56:51.186 STEP: Verify that aggregate-to-admin ClusterRole is created. @ 10/30/24 13:56:51.188 STEP: Verify that ClusterRole has correct labels and annotations. @ 10/30/24 13:56:51.203 STEP: Verify that ClusterRole has correct policy rules. @ 10/30/24 13:56:51.203 STEP: Verify that aggregate-to-edit ClusterRole is created. @ 10/30/24 13:56:51.203 STEP: Verify that ClusterRole has correct labels and annotations. @ 10/30/24 13:56:51.216 STEP: Verify that ClusterRole has correct policy rules. @ 10/30/24 13:56:51.216 STEP: Verify that aggregate-to-view ClusterRole is created. @ 10/30/24 13:56:51.216 STEP: Verify that ClusterRole has correct labels and annotations. @ 10/30/24 13:56:51.229 STEP: Verify that ClusterRole has correct policy rules. @ 10/30/24 13:56:51.229 STEP: Verify that ClusterRoleBinding is created. @ 10/30/24 13:56:51.229 STEP: Verify that ClusterRoleBinding has correct labels and annotations. @ 10/30/24 13:56:51.242 [FAILED] in [It] - /tmp/tmp.mWLvz2au9x/argo-rollouts-manager/tests/e2e/test_utils.go:163 @ 10/30/24 13:56:51.243 << Timeline [FAILED] Expected : { "app.kubernetes.io/component": "argo-rollouts", "app.kubernetes.io/name": "argo-rollouts", "app.kubernetes.io/part-of": "argo-rollouts", } to have {key: value} : { "foo-label2": "bar-label2", } In [It] at: /tmp/tmp.mWLvz2au9x/argo-rollouts-manager/tests/e2e/test_utils.go:163 @ 10/30/24 13:56:51.243 ------------------------------ HavePhase: expected: Available actual: HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Available HaveSuccessCondition: expected: {Reconciled True 0 0001-01-01 00:00:00 +0000 UTC Success } actual: {Reconciled True 0 2024-10-30 13:57:01 +0000 UTC Success } Object exists in ExistByName: argo-rollouts Object exists in ExistByName: argo-rollouts Object exists in ExistByName: argo-rollouts-aggregate-to-admin Object exists in ExistByName: argo-rollouts-aggregate-to-edit Object exists in ExistByName: argo-rollouts-aggregate-to-view Object exists in ExistByName: argo-rollouts Object exists in ExistByName: argo-rollouts-metrics Object exists in ExistByName: argo-rollouts-notification-secret Object exists in ExistByName: argo-rollouts Object exists in ExistByName: rollout-bluegreen-active Object exists in ExistByName: rollout-bluegreen-preview Object exists in ExistByName: rollout-bluegreen-active Object exists in ExistByName: rollout-bluegreen-preview •HavePhase: expected: Available actual: HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Available HaveSuccessCondition: expected: {Reconciled True 0 0001-01-01 00:00:00 +0000 UTC Success } actual: {Reconciled True 0 2024-10-30 13:58:15 +0000 UTC Success } Object exists in ExistByName: rollout-bluegreen-active Object exists in ExistByName: rollout-bluegreen-preview HavePhase: expected: Failure actual: Failure HaveCondition: expected: {Reconciled False 0 0001-01-01 00:00:00 +0000 UTC InvalidRolloutManagerScope when Subscription has environment variable NAMESPACE_SCOPED_ARGO_ROLLOUTS set to False, there may not exist any namespace-scoped RolloutManagers: only a single cluster-scoped RolloutManager is supported} actual: {Reconciled False 0 2024-10-30 13:58:33 +0000 UTC InvalidRolloutManagerScope when Subscription has environment variable NAMESPACE_SCOPED_ARGO_ROLLOUTS set to False, there may not exist any namespace-scoped RolloutManagers: only a single cluster-scoped RolloutManager is supported} Object exists in ExistByName: rollout-bluegreen-active Object exists in ExistByName: rollout-bluegreen-preview HavePhase: expected: Available actual: Available HaveSuccessCondition: expected: {Reconciled True 0 0001-01-01 00:00:00 +0000 UTC Success } actual: {Reconciled True 0 2024-10-30 13:58:15 +0000 UTC Success } Object exists in ExistByName: rollout-bluegreen-active Object exists in ExistByName: rollout-bluegreen-preview •HavePhase: expected: Available actual: HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Available HaveSuccessCondition: expected: {Reconciled True 0 0001-01-01 00:00:00 +0000 UTC Success } actual: {Reconciled True 0 2024-10-30 13:59:18 +0000 UTC Success } Object exists in ExistByName: rollout-bluegreen-active Object exists in ExistByName: rollout-bluegreen-preview HavePhase: expected: Failure actual: Failure HaveCondition: expected: {Reconciled False 0 0001-01-01 00:00:00 +0000 UTC MultipleClusterScopedRolloutManager when there exists a cluster-scoped RolloutManager on the cluster, there may not exist another: only a single cluster-scoped RolloutManager is supported} actual: {Reconciled False 0 2024-10-30 13:59:32 +0000 UTC MultipleClusterScopedRolloutManager when there exists a cluster-scoped RolloutManager on the cluster, there may not exist another: only a single cluster-scoped RolloutManager is supported} Object exists in ExistByName: rollout-bluegreen-active Object exists in ExistByName: rollout-bluegreen-preview HavePhase: expected: Failure actual: Failure HaveCondition: expected: {Reconciled False 0 0001-01-01 00:00:00 +0000 UTC MultipleClusterScopedRolloutManager when there exists a cluster-scoped RolloutManager on the cluster, there may not exist another: only a single cluster-scoped RolloutManager is supported} actual: {Reconciled False 0 2024-10-30 13:59:34 +0000 UTC MultipleClusterScopedRolloutManager when there exists a cluster-scoped RolloutManager on the cluster, there may not exist another: only a single cluster-scoped RolloutManager is supported} Object exists in ExistByName: rollout-bluegreen-active Object exists in ExistByName: rollout-bluegreen-preview •HavePhase: expected: Failure actual: Failure HaveCondition: expected: {Reconciled False 0 0001-01-01 00:00:00 +0000 UTC InvalidRolloutManagerNamespace Namespace is not specified in CLUSTER_SCOPED_ARGO_ROLLOUTS_NAMESPACES environment variable of Subscription resource. If you wish to install a cluster-scoped Argo Rollouts instance outside the default namespace, ensure it is defined in CLUSTER_SCOPED_ARGO_ROLLOUTS_NAMESPACES} actual: {Reconciled False 0 2024-10-30 14:00:12 +0000 UTC InvalidRolloutManagerNamespace Namespace is not specified in CLUSTER_SCOPED_ARGO_ROLLOUTS_NAMESPACES environment variable of Subscription resource. If you wish to install a cluster-scoped Argo Rollouts instance outside the default namespace, ensure it is defined in CLUSTER_SCOPED_ARGO_ROLLOUTS_NAMESPACES} •HavePhase: expected: Available actual: HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Pending HavePhase: expected: Available actual: Available • Summarizing 1 Failure: [FAIL] Cluster-scoped RolloutManager tests Testing cluster-scoped RolloutManager behaviour [It] After creating cluster-scoped RolloutManager in one namespace, operator should create appropriate K8s resources and watch argo rollouts CR in another namespace. /tmp/tmp.mWLvz2au9x/argo-rollouts-manager/tests/e2e/test_utils.go:163 Ran 6 of 6 Specs in 264.586 seconds FAIL! -- 5 Passed | 1 Failed | 0 Pending | 0 Skipped --- FAIL: TestClusterScoped (264.59s) FAIL coverage: [no statements] FAIL github.com/argoproj-labs/argo-rollouts-manager/tests/e2e/cluster-scoped 264.713s FAIL make: *** [Makefile:120: test-e2e] Error 1 + exit_code=2 + [[ 2 != 0 ]] + echo 2 + tee /tekton/results/result 2 + failed=2 + exit_code=0 + kubectl delete rolloutmanagers --all -n test-rom-ns-1 No resources found + cd /tmp/tmp.mWLvz2au9x/argo-rollouts-manager + SKIP_RUN_STEP=true + hack/run-upstream-argo-rollouts-e2e-tests.sh + set -e + trap cleanup EXIT +++ dirname -- hack/run-upstream-argo-rollouts-e2e-tests.sh ++ cd -- hack ++ pwd + SCRIPT_DIR=/tmp/tmp.mWLvz2au9x/argo-rollouts-manager/hack ++ mktemp -d + TMP_DIR=/tmp/tmp.SIcIuinRDW + cd /tmp/tmp.SIcIuinRDW + git clone https://github.com/argoproj/argo-rollouts Cloning into 'argo-rollouts'... + cd argo-rollouts + git checkout v1.6.6 Note: switching to 'v1.6.6'. You are in 'detached HEAD' state. You can look around, make experimental changes and commit them, and you can discard any commits you make in this state without impacting any branches by switching back to a branch. If you want to create a new branch to retain commits you create, you may do so (now or later) by using -c with the switch command. Example: git switch -c Or undo this operation with: git switch - Turn off this advice by setting config variable advice.detachedHead to false HEAD is now at 737ca89b4 fix: stuck rollout when 2nd deployment happens before 1st finishes (#3354) + go mod tidy go: downloading github.com/argoproj/notifications-engine v0.4.1-0.20231011160156-2d2d1a75dbee go: downloading github.com/spf13/cobra v1.7.0 go: downloading k8s.io/apimachinery v0.25.8 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading k8s.io/cli-runtime v0.25.8 go: downloading k8s.io/client-go v0.25.8 go: downloading k8s.io/klog/v2 v2.80.1 go: downloading github.com/argoproj/pkg v0.13.6 go: downloading github.com/servicemeshinterface/smi-sdk-go v0.5.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading k8s.io/code-generator v0.25.8 go: downloading k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 go: downloading k8s.io/kubernetes v1.25.8 go: downloading github.com/stretchr/testify v1.8.4 go: downloading github.com/blang/semver v3.5.1+incompatible go: downloading k8s.io/apiextensions-apiserver v0.25.8 go: downloading github.com/hashicorp/go-plugin v1.4.10 go: downloading k8s.io/api v0.25.8 go: downloading k8s.io/component-base v0.25.8 go: downloading github.com/aws/aws-sdk-go-v2/config v1.19.1 go: downloading github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.27.2 go: downloading github.com/bombsimon/logrusr/v4 v4.0.0 go: downloading k8s.io/kubectl v0.25.8 go: downloading github.com/aws/aws-sdk-go-v2 v1.21.2 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/prometheus/client_model v0.3.0 go: downloading golang.org/x/sys v0.10.0 go: downloading github.com/inconshreveable/mousetrap v1.1.0 go: downloading k8s.io/gengo v0.0.0-20220902162205-c0856e24416d go: downloading golang.org/x/tools v0.6.0 go: downloading github.com/golang/glog v1.1.0 go: downloading github.com/antonmedv/expr v1.15.3 go: downloading github.com/influxdata/influxdb-client-go/v2 v2.12.3 go: downloading github.com/newrelic/newrelic-client-go v1.1.0 go: downloading github.com/tj/assert v0.0.3 go: downloading github.com/prometheus/common v0.42.0 go: downloading github.com/prometheus/common/sigv4 v0.1.0 go: downloading github.com/machinebox/graphql v0.2.2 go: downloading github.com/spaceapegames/go-wavefront v1.8.1 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 go: downloading google.golang.org/grpc v1.57.0 go: downloading google.golang.org/protobuf v1.31.0 go: downloading google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 go: downloading gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c go: downloading github.com/juju/ansiterm v1.0.0 go: downloading github.com/pmezard/go-difflib v1.0.0 go: downloading github.com/go-openapi/jsonreference v0.20.0 go: downloading github.com/go-openapi/swag v0.21.1 go: downloading github.com/google/gnostic v0.6.9 go: downloading github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.21.0 go: downloading github.com/mitchellh/mapstructure v1.5.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading k8s.io/apiserver v0.25.8 go: downloading github.com/hashicorp/go-hclog v0.14.1 go: downloading github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb go: downloading github.com/mitchellh/go-testing-interface v1.0.0 go: downloading github.com/oklog/run v1.0.0 go: downloading github.com/jhump/protoreflect v1.6.0 go: downloading github.com/Azure/go-autorest/autorest v0.11.27 go: downloading github.com/Azure/go-autorest v14.2.0+incompatible go: downloading github.com/Azure/go-autorest/autorest/adal v0.9.20 go: downloading github.com/stretchr/objx v0.5.0 go: downloading golang.org/x/term v0.10.0 go: downloading github.com/valyala/fasttemplate v1.2.2 go: downloading golang.org/x/net v0.12.0 go: downloading github.com/PagerDuty/go-pagerduty v1.7.0 go: downloading github.com/RocketChat/Rocket.Chat.Go.SDK v0.0.0-20220708192748-b73dcb041214 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.43 go: downloading github.com/aws/aws-sdk-go-v2/service/sqs v1.20.0 go: downloading github.com/bradleyfalzon/ghinstallation/v2 v2.5.0 go: downloading github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 go: downloading github.com/google/go-github/v41 v41.0.0 go: downloading github.com/gregdel/pushover v1.2.1 go: downloading github.com/opsgenie/opsgenie-go-sdk-v2 v1.2.13 go: downloading github.com/slack-go/slack v0.12.2 go: downloading github.com/spf13/cast v1.5.1 go: downloading github.com/whilp/git-urls v0.0.0-20191001220047-6db9661140c0 go: downloading gomodules.xyz/notify v0.1.1 go: downloading google.golang.org/api v0.132.0 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 go: downloading github.com/aws/smithy-go v1.15.0 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 go: downloading github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de go: downloading golang.org/x/text v0.11.0 go: downloading sigs.k8s.io/kustomize/api v0.12.1 go: downloading sigs.k8s.io/kustomize/kyaml v0.13.9 go: downloading github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 go: downloading github.com/peterbourgon/diskv v2.0.1+incompatible go: downloading github.com/emicklei/go-restful/v3 v3.10.1 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 go: downloading github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 go: downloading github.com/deepmap/oapi-codegen v1.11.0 go: downloading github.com/felixge/httpsnoop v1.0.3 go: downloading github.com/matryer/is v1.4.0 go: downloading github.com/aws/aws-sdk-go v1.44.116 go: downloading github.com/elazarl/goproxy v0.0.0-20220417044921-416226498f94 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/mattn/go-colorable v0.1.12 go: downloading github.com/mattn/go-isatty v0.0.14 go: downloading github.com/go-openapi/jsonpointer v0.19.5 go: downloading github.com/Masterminds/sprig/v3 v3.2.3 go: downloading github.com/fatih/color v1.7.0 go: downloading github.com/Azure/go-autorest/logger v0.2.1 go: downloading github.com/Azure/go-autorest/tracing v0.6.0 go: downloading github.com/Azure/go-autorest/autorest/mocks v0.4.2 go: downloading github.com/Azure/go-autorest/autorest/date v0.3.0 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading golang.org/x/crypto v0.11.0 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading google.golang.org/appengine v1.6.7 go: downloading github.com/valyala/bytebufferpool v1.0.0 go: downloading github.com/onsi/ginkgo/v2 v2.1.6 go: downloading github.com/onsi/gomega v1.20.1 go: downloading github.com/google/go-github/v53 v53.0.0 go: downloading cloud.google.com/go/compute v1.20.1 go: downloading github.com/google/go-querystring v1.1.0 go: downloading github.com/hashicorp/go-retryablehttp v0.7.1 go: downloading github.com/frankban/quicktest v1.14.4 go: downloading gomodules.xyz/envconfig v1.3.1-0.20190308184047-426f31af0d45 go: downloading gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df go: downloading github.com/blang/semver/v4 v4.0.0 go: downloading github.com/gorilla/websocket v1.5.0 go: downloading github.com/go-test/deep v1.0.4 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 go: downloading github.com/jmespath/go-jmespath/internal/testify v1.5.1 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f go: downloading github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf go: downloading github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f go: downloading github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 go: downloading github.com/valyala/fastjson v1.6.3 go: downloading github.com/lunixbochs/vtclean v1.0.0 go: downloading k8s.io/component-helpers v0.25.8 go: downloading golang.org/x/mod v0.8.0 go: downloading github.com/chai2010/gettext-go v1.0.2 go: downloading github.com/MakeNowJust/heredoc v1.0.0 go: downloading github.com/mitchellh/go-wordwrap v1.0.1 go: downloading github.com/russross/blackfriday v1.6.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.9.0 go: downloading github.com/Masterminds/goutils v1.1.1 go: downloading github.com/Masterminds/semver/v3 v3.2.0 go: downloading github.com/onsi/ginkgo v1.14.1 go: downloading github.com/huandu/xstrings v1.3.3 go: downloading github.com/mitchellh/copystructure v1.2.0 go: downloading github.com/shopspring/decimal v1.2.0 go: downloading github.com/Jeffail/gabs v1.4.0 go: downloading github.com/gopackage/ddp v0.0.0-20170117053602-652027933df4 go: downloading github.com/sony/sonyflake v1.0.0 go: downloading github.com/hashicorp/go-cleanhttp v0.5.2 go: downloading gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc go: downloading github.com/go-errors/errors v1.4.2 go: downloading github.com/jpillora/backoff v1.0.0 go: downloading k8s.io/cluster-bootstrap v0.25.8 go: downloading github.com/docker/distribution v2.8.1+incompatible go: downloading golang.org/x/sync v0.3.0 go: downloading github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 go: downloading github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae go: downloading github.com/mitchellh/reflectwalk v1.0.2 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 go: downloading github.com/xlab/treeprint v1.1.0 go: downloading github.com/creack/pty v1.1.11 go: downloading gotest.tools/v3 v3.0.3 go: downloading github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 go: downloading github.com/moby/spdystream v0.2.0 go: downloading github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 go: downloading go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 go: downloading github.com/sergi/go-diff v1.1.0 go: downloading github.com/cloudflare/circl v1.3.3 go: downloading github.com/opencontainers/go-digest v1.0.0 go: downloading github.com/googleapis/gax-go/v2 v2.12.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.2.5 go: downloading go.opencensus.io v0.24.0 go: downloading github.com/google/s2a-go v0.1.4 + kubectl delete ns argo-rollouts namespace "argo-rollouts" deleted + kubectl wait --timeout=5m --for=delete namespace/argo-rollouts + kubectl create ns argo-rollouts namespace/argo-rollouts created + kubectl config set-context --current --namespace=argo-rollouts Context "default/api-psi-416-ocp-gitops-qe-com:6443/kube:admin" modified. + cd /tmp/tmp.mWLvz2au9x/argo-rollouts-manager/hack/.. + '[' -z true ']' + cd /tmp/tmp.SIcIuinRDW/argo-rollouts + cat + kubectl apply -f /tmp/tmp.SIcIuinRDW/rollout-manager.yaml rolloutmanager.argoproj.io/argo-rollout created + echo '* Waiting for Argo Rollouts Deployment to exist' * Waiting for Argo Rollouts Deployment to exist + kubectl get -n argo-rollouts deployment/argo-rollouts NAME READY UP-TO-DATE AVAILABLE AGE argo-rollouts 0/1 1 0 1s + kubectl wait --for=condition=Available --timeout=10m -n argo-rollouts deployment/argo-rollouts deployment.apps/argo-rollouts condition met + kubectl apply -f test/e2e/crds customresourcedefinition.apiextensions.k8s.io/apisixroutes.apisix.apache.org configured customresourcedefinition.apiextensions.k8s.io/wasmplugins.extensions.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/destinationrules.networking.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/envoyfilters.networking.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/gateways.networking.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/proxyconfigs.networking.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/serviceentries.networking.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/sidecars.networking.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/virtualservices.networking.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/workloadentries.networking.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/workloadgroups.networking.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/authorizationpolicies.security.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/peerauthentications.security.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/requestauthentications.security.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/telemetries.telemetry.istio.io unchanged customresourcedefinition.apiextensions.k8s.io/trafficsplits.split.smi-spec.io unchanged + oc adm policy add-scc-to-user anyuid -z argo-rollouts -n argo-rollouts clusterrole.rbac.authorization.k8s.io/system:openshift:scc:anyuid added: "argo-rollouts" + oc adm policy add-scc-to-user anyuid -z default -n argo-rollouts clusterrole.rbac.authorization.k8s.io/system:openshift:scc:anyuid added: "default" + rm -f /tmp/test-e2e.log + set +e + make test-e2e + tee /tmp/test-e2e.log ./hack/installers/install-dev-tools.sh +++ dirname ./hack/installers/install-dev-tools.sh ++ cd ./hack/installers/../.. ++ pwd + PROJECT_ROOT=/tmp/tmp.SIcIuinRDW/argo-rollouts + DIST_PATH=/tmp/tmp.SIcIuinRDW/argo-rollouts/dist + PATH=/tmp/tmp.SIcIuinRDW/argo-rollouts/dist:/tmp/tmp.SIcIuinRDW/argo-rollouts/dist:/root/.nvm/versions/node/v20.12.2/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin + mkdir -p /tmp/tmp.SIcIuinRDW/argo-rollouts/dist + gotestsum_version=1.10.1 ++ go env GOOS + OS=linux ++ go env GOARCH + ARCH=amd64 + export TARGET_FILE=gotestsum_1.10.1_linux_amd64.tar.gz + TARGET_FILE=gotestsum_1.10.1_linux_amd64.tar.gz + temp_path=/tmp/gotestsum_1.10.1_linux_amd64.tar.gz + url=https://github.com/gotestyourself/gotestsum/releases/download/v1.10.1/gotestsum_1.10.1_linux_amd64.tar.gz + '[' -e /tmp/gotestsum_1.10.1_linux_amd64.tar.gz ']' + curl -sLf --retry 3 -o /tmp/gotestsum_1.10.1_linux_amd64.tar.gz https://github.com/gotestyourself/gotestsum/releases/download/v1.10.1/gotestsum_1.10.1_linux_amd64.tar.gz + mkdir -p /tmp/gotestsum-1.10.1 + tar -xvzf /tmp/gotestsum_1.10.1_linux_amd64.tar.gz -C /tmp/gotestsum-1.10.1 LICENSE LICENSE.md README.md gotestsum + cp /tmp/gotestsum-1.10.1/gotestsum /tmp/tmp.SIcIuinRDW/argo-rollouts/dist/gotestsum + chmod +x /tmp/tmp.SIcIuinRDW/argo-rollouts/dist/gotestsum + gotestsum --version gotestsum version 1.10.1 /tmp/tmp.SIcIuinRDW/argo-rollouts/dist/gotestsum --rerun-fails-report=rerunreport.txt --junitfile=junit.xml --format=testname --packages="./test/e2e" --rerun-fails=5 -- -timeout 60m -count 1 --tags e2e -p 1 -parallel 1 -v --short ./test/e2e PASS test/e2e.TestAnalysisSuite/TestAnalysisWithArgs (42.04s) PASS test/e2e.TestAnalysisSuite/TestAnalysisWithSecret (31.97s) PASS test/e2e.TestAnalysisSuite/TestBackgroundAnalysisWithArgs (31.13s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenAbortAndUpdate (22.25s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenAnalysis (30.96s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenKitchenSink (47.44s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenPostPromotionFail (15.72s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenPrePromotionFail (28.33s) PASS test/e2e.TestAnalysisSuite/TestCanaryBackgroundAnalysis (4.74s) PASS test/e2e.TestAnalysisSuite/TestCanaryFailInlineMultipleAnalysis (13.05s) PASS test/e2e.TestAnalysisSuite/TestCanaryInlineAnalysis (15.06s) PASS test/e2e.TestAnalysisSuite/TestCanaryInlineMultipleAnalysis (17.07s) PASS test/e2e.TestAnalysisSuite/TestMultipleAnalysis (9.89s) PASS test/e2e.TestAnalysisSuite (323.17s) === RUN TestAPISIXSuite/TestAPISIXCanarySetHeaderStep time="2024-10-30T14:18:32Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:18:34Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-10-30T14:18:36Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-10-30T14:18:37Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route created\n" rollout=rollout-apisix-canary time="2024-10-30T14:18:39Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:18:39Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-10-30T14:18:39Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (7.60s) FAIL test/e2e.TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (7.60s) PASS test/e2e.TestAPISIXSuite/TestAPISIXCanaryStep (28.36s) === RUN TestAPISIXSuite --- FAIL: TestAPISIXSuite (36.31s) FAIL test/e2e.TestAPISIXSuite (36.31s) PASS test/e2e.TestAWSSuite/TestALBExperimentStep (24.92s) PASS test/e2e.TestAWSSuite/TestALBExperimentStepMultiIngress (25.13s) PASS test/e2e.TestAWSSuite/TestALBExperimentStepNoSetWeight (23.96s) === RUN TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress time="2024-10-30T14:20:24Z" level=info msg="Deleting e2e-test-name=TestALBExperimentStepNoSetWeightMultiIngress" time="2024-10-30T14:20:26Z" level=info msg="service/alb-rollout-root created\n" rollout=alb-rollout time="2024-10-30T14:20:27Z" level=info msg="service/alb-rollout-canary created\n" rollout=alb-rollout time="2024-10-30T14:20:28Z" level=info msg="service/alb-rollout-stable created\n" rollout=alb-rollout time="2024-10-30T14:20:29Z" level=info msg="Warning: annotation \"kubernetes.io/ingress.class\" is deprecated, please use 'spec.ingressClassName' instead\ningress.networking.k8s.io/alb-rollout-multi-ingress-1 created\n" rollout=alb-rollout time="2024-10-30T14:20:30Z" level=info msg="Warning: annotation \"kubernetes.io/ingress.class\" is deprecated, please use 'spec.ingressClassName' instead\ningress.networking.k8s.io/alb-rollout-multi-ingress-2 created\n" rollout=alb-rollout time="2024-10-30T14:20:31Z" level=info msg="rollout.argoproj.io/alb-rollout created\n" rollout=alb-rollout time="2024-10-30T14:20:31Z" level=info msg="Waiting for condition: status=Healthy" rollout=alb-rollout time="2024-10-30T14:20:32Z" level=info msg="Condition 'status=Healthy' met after 0s" rollout=alb-rollout time="2024-10-30T14:20:32Z" level=info msg="Experiment expectation 'experiment count == 0' met" rollout=alb-rollout time="2024-10-30T14:20:32Z" level=info msg="Updated rollout pod spec: 2024-10-30T14:20:32.566524227Z" rollout=alb-rollout time="2024-10-30T14:20:32Z" level=info msg="Sleeping 10s" rollout=alb-rollout require.go:72: Error Trace: /root/go/pkg/mod/github.com/tj/assert@v0.0.3/require.go:72 /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/aws_test.go:298 /tmp/tmp.SIcIuinRDW/argo-rollouts/test/fixtures/then.go:28 /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/aws_test.go:287 Error: Not equal: expected: "{\"Type\":\"forward\",\"ForwardConfig\":{\"TargetGroups\":[{\"ServiceName\":\"alb-rollout-canary\",\"ServicePort\":\"80\",\"Weight\":0},{\"ServiceName\":\"\",\"ServicePort\":\"80\",\"Weight\":20},{\"ServiceName\":\"\",\"ServicePort\":\"80\",\"Weight\":20},{\"ServiceName\":\"alb-rollout-stable\",\"ServicePort\":\"80\",\"Weight\":60}]}}" actual : "{\"Type\":\"forward\",\"ForwardConfig\":{\"TargetGroups\":[{\"ServiceName\":\"alb-rollout-canary\",\"ServicePort\":\"80\",\"Weight\":0},{\"ServicePort\":\"80\",\"Weight\":20},{\"ServicePort\":\"80\",\"Weight\":20},{\"ServiceName\":\"alb-rollout-stable\",\"ServicePort\":\"80\",\"Weight\":60}]}}" Diff: --- Expected +++ Actual @@ -1 +1 @@ -{"Type":"forward","ForwardConfig":{"TargetGroups":[{"ServiceName":"alb-rollout-canary","ServicePort":"80","Weight":0},{"ServiceName":"","ServicePort":"80","Weight":20},{"ServiceName":"","ServicePort":"80","Weight":20},{"ServiceName":"alb-rollout-stable","ServicePort":"80","Weight":60}]}} +{"Type":"forward","ForwardConfig":{"TargetGroups":[{"ServiceName":"alb-rollout-canary","ServicePort":"80","Weight":0},{"ServicePort":"80","Weight":20},{"ServicePort":"80","Weight":20},{"ServiceName":"alb-rollout-stable","ServicePort":"80","Weight":60}]}} Test: TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress Name: alb-rollout Namespace: argo-rollouts Status: ◌ Progressing Message: more replicas need to be updated Strategy: Canary Step: 0/1 SetWeight: 0 ActualWeight: 0 Images: nginx:1.19-alpine (stable, Σ:experiment-alb-canary, Σ:experiment-alb-stable) Replicas: Desired: 1 Current: 1 Updated: 0 Ready: 1 Available: 1 NAME KIND STATUS AGE INFO ⟳ alb-rollout Rollout ◌ Progressing 11s ├──# revision:2 │ ├──⧉ alb-rollout-6bc4748d8c ReplicaSet • ScaledDown 10s canary │ └──Σ alb-rollout-6bc4748d8c-2-0 Experiment ◌ Running 10s │ ├──⧉ alb-rollout-6bc4748d8c-2-0-experiment-alb-canary ReplicaSet ✔ Healthy 10s │ │ └──□ alb-rollout-6bc4748d8c-2-0-experiment-alb-canary-46cnd Pod ✔ Running 10s ready:1/1 │ └──⧉ alb-rollout-6bc4748d8c-2-0-experiment-alb-stable ReplicaSet ✔ Healthy 10s │ └──□ alb-rollout-6bc4748d8c-2-0-experiment-alb-stable-gzvk9 Pod ✔ Running 10s ready:1/1 └──# revision:1 └──⧉ alb-rollout-d6f4fdcdc ReplicaSet ✔ Healthy 11s stable └──□ alb-rollout-d6f4fdcdc-glgpk Pod ✔ Running 11s ready:1/1 --- apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: annotations: rollout.argoproj.io/revision: "2" creationTimestamp: "2024-10-30T14:20:31Z" generation: 3 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestALBExperimentStepNoSetWeightMultiIngress name: alb-rollout namespace: argo-rollouts resourceVersion: "10254495" uid: d47c7a56-cd3c-4b0e-a157-8f63888ebd57 spec: replicas: 1 selector: matchLabels: app: alb-rollout strategy: canary: canaryService: alb-rollout-canary stableService: alb-rollout-stable steps: - experiment: duration: 15s templates: - metadata: {} name: experiment-alb-canary specRef: canary weight: 20 - metadata: {} name: experiment-alb-stable specRef: stable weight: 20 trafficRouting: alb: ingresses: - alb-rollout-multi-ingress-1 - alb-rollout-multi-ingress-2 rootService: alb-rollout-root servicePort: 80 template: metadata: annotations: update: "2024-10-30T14:20:32.566524227Z" creationTimestamp: null labels: app: alb-rollout spec: containers: - image: nginx:1.19-alpine name: alb-rollout ports: - containerPort: 80 name: http protocol: TCP resources: requests: cpu: 5m memory: 16Mi status: HPAReplicas: 1 availableReplicas: 1 blueGreen: {} canary: currentExperiment: alb-rollout-6bc4748d8c-2-0 weights: additional: - weight: 20 - weight: 20 canary: podTemplateHash: 6bc4748d8c serviceName: alb-rollout-canary weight: 0 stable: podTemplateHash: d6f4fdcdc serviceName: alb-rollout-stable weight: 60 conditions: - lastTransitionTime: "2024-10-30T14:20:32Z" lastUpdateTime: "2024-10-30T14:20:32Z" message: Rollout has minimum availability reason: AvailableReason status: "True" type: Available - lastTransitionTime: "2024-10-30T14:20:32Z" lastUpdateTime: "2024-10-30T14:20:32Z" message: Rollout is not healthy reason: RolloutHealthy status: "False" type: Healthy - lastTransitionTime: "2024-10-30T14:20:32Z" lastUpdateTime: "2024-10-30T14:20:32Z" message: RolloutCompleted reason: RolloutCompleted status: "False" type: Completed - lastTransitionTime: "2024-10-30T14:20:31Z" lastUpdateTime: "2024-10-30T14:20:32Z" message: Created new replica set "alb-rollout-6bc4748d8c" reason: NewReplicaSetCreated status: "True" type: Progressing currentPodHash: 6bc4748d8c currentStepHash: 558cc8bf4d currentStepIndex: 0 message: more replicas need to be updated observedGeneration: "3" phase: Progressing readyReplicas: 1 replicas: 1 selector: app=alb-rollout stableRS: d6f4fdcdc 2024-10-30T14:20:31Z Normal RolloutAddedToInformer Rollout resource added to informer: argo-rollouts/alb-rollout 2024-10-30T14:20:31Z Normal RolloutNotCompleted Rollout not completed, started update to revision 1 (d6f4fdcdc) 2024-10-30T14:20:31Z Normal RolloutUpdated Rollout updated to revision 1 2024-10-30T14:20:31Z Normal NewReplicaSetCreated Created ReplicaSet alb-rollout-d6f4fdcdc (revision 1) 2024-10-30T14:20:34Z (x2) Normal PatchingALBIngress Updating Ingress `alb-rollout-multi-ingress-1` to desiredWeight '0' 2024-10-30T14:20:34Z (x2) Normal PatchingALBIngress Updating Ingress `alb-rollout-multi-ingress-2` to desiredWeight '0' 2024-10-30T14:20:31Z (x2) Normal TrafficWeightUpdated Traffic weight updated to 0 2024-10-30T14:20:31Z Normal ScalingReplicaSet Scaled up ReplicaSet alb-rollout-d6f4fdcdc (revision 1) from 0 to 1 2024-10-30T14:20:31Z Normal RolloutCompleted Rollout completed update to revision 1 (d6f4fdcdc): Initial deploy 2024-10-30T14:20:32Z (x2) Normal TrafficWeightUpdated Traffic weight updated 2024-10-30T14:20:32Z Normal SwitchService Switched selector for service 'alb-rollout-stable' from '' to 'd6f4fdcdc' 2024-10-30T14:20:32Z Normal SwitchService Switched selector for service 'alb-rollout-canary' from '' to 'd6f4fdcdc' 2024-10-30T14:20:32Z Normal RolloutNotCompleted Rollout not completed, started update to revision 2 (6bc4748d8c) 2024-10-30T14:20:32Z Normal RolloutUpdated Rollout updated to revision 2 2024-10-30T14:20:32Z Normal NewReplicaSetCreated Created ReplicaSet alb-rollout-6bc4748d8c (revision 2) 2024-10-30T14:20:32Z Normal ExperimentCreated Created Experiment 'alb-rollout-6bc4748d8c-2-0' 2024-10-30T14:20:34Z Normal TrafficWeightUpdated Traffic weight updated additional: [{20 } {20 }] time="2024-10-30T14:20:42Z" level=info msg="Deleting e2e-test-name=TestALBExperimentStepNoSetWeightMultiIngress" --- FAIL: TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress (18.48s) FAIL test/e2e.TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress (18.48s) PASS test/e2e.TestAWSSuite/TestALBPingPongUpdate (15.72s) PASS test/e2e.TestAWSSuite/TestALBPingPongUpdateMultiIngress (15.09s) PASS test/e2e.TestAWSSuite/TestAlbHeaderRoute (13.28s) PASS test/e2e.TestAWSSuite/TestAlbHeaderRouteMultiIngress (25.18s) === RUN TestAWSSuite --- FAIL: TestAWSSuite (163.89s) FAIL test/e2e.TestAWSSuite (163.89s) PASS test/e2e.TestBlueGreenSuite/TestBlueGreenPreviewReplicaCount (7.26s) PASS test/e2e.TestBlueGreenSuite/TestBlueGreenProgressDeadlineExceededWithPause (63.88s) PASS test/e2e.TestBlueGreenSuite/TestBlueGreenProgressDeadlineExceededWithoutPause (63.88s) PASS test/e2e.TestBlueGreenSuite/TestEphemeralMetadata (9.84s) PASS test/e2e.TestBlueGreenSuite (145.28s) PASS test/e2e.TestCanarySuite/TestCanaryDynamicStableScale (23.54s) PASS test/e2e.TestCanarySuite/TestCanaryDynamicStableScaleRollbackToStable (17.24s) PASS test/e2e.TestCanarySuite/TestCanaryProgressDeadlineExceededWithPause (123.69s) PASS test/e2e.TestCanarySuite/TestCanaryScaleDownDelay (14.38s) PASS test/e2e.TestCanarySuite/TestCanaryScaleDownOnAbort (12.00s) PASS test/e2e.TestCanarySuite/TestCanaryScaleDownOnAbortNoTrafficRouting (11.11s) PASS test/e2e.TestCanarySuite/TestCanaryUnScaleDownOnAbort (11.67s) PASS test/e2e.TestCanarySuite/TestCanaryWithPausedRollout (102.16s) PASS test/e2e.TestCanarySuite/TestEphemeralMetadata (7.10s) PASS test/e2e.TestCanarySuite/TestReduceWeightAndHonorMaxUnavailable (27.07s) PASS test/e2e.TestCanarySuite/TestRolloutScalingDuringUpdate (4.45s) PASS test/e2e.TestCanarySuite/TestRolloutScalingWhenPaused (6.43s) PASS test/e2e.TestCanarySuite/TestRolloutWithMaxSurgeScalingDuringUpdate (3.53s) PASS test/e2e.TestCanarySuite/TestCanarySetCanaryScale (12.70s) PASS test/e2e.TestCanarySuite (366.27s) === RUN TestExperimentSuite/TestExperimentWithDryRunMetrics time="2024-10-30T14:30:36Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:30:38Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-10-30T14:30:39Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-10-30T14:30:39Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-10-30T14:31:10Z" level=info msg="Condition 'phase=Successful' met after 30s" time="2024-10-30T14:31:10Z" level=info msg="Sleeping 3s" time="2024-10-30T14:31:13Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-10-30T14:31:13Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-10-30T14:31:13Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 34s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 34s delay:26s │ └──□ experiment-with-dry-run-baseline-r8p2q Pod ✔ Running 34s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ 403c550f-f53b-4b1b-a0a9-3af4f099a7a8.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-10-30T14:30:39Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "10264901" uid: 0e58c5a1-ab85-4241-abd0-0e7a86e7ed4c spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-10-30T14:30:40Z" conditions: - lastTransitionTime: "2024-10-30T14:31:10Z" lastUpdateTime: "2024-10-30T14:31:10Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-10-30T14:31:10Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-10-30T14:30:39Z Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-10-30T14:30:39Z Normal ExperimentPending Experiment transitioned from -> Pending 2024-10-30T14:30:39Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-10-30T14:30:40Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-10-30T14:30:40Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-10-30T14:30:40Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-10-30T14:30:40Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-10-30T14:31:10Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-10-30T14:31:10Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-10-30T14:31:10Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-10-30T14:31:13Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (36.79s) FAIL test/e2e.TestExperimentSuite/TestExperimentWithDryRunMetrics (36.79s) PASS test/e2e.TestExperimentSuite/TestExperimentWithMeasurementRetentionMetrics (36.77s) PASS test/e2e.TestExperimentSuite/TestExperimentWithMultiportServiceAndScaleDownDelay (18.08s) PASS test/e2e.TestExperimentSuite/TestExperimentWithServiceAndScaleDownDelay (17.97s) PASS test/e2e.TestExperimentSuite/TestExperimentWithServiceNameAndScaleDownDelay (17.07s) PASS test/e2e.TestExperimentSuite/TestRolloutWithExperimentAndAnalysis (10.27s) === RUN TestExperimentSuite --- FAIL: TestExperimentSuite (127.36s) FAIL test/e2e.TestExperimentSuite (127.36s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenExceedProgressDeadlineAbort (10.26s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenPreviewReplicaCountPromoteFull (8.63s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenPromoteFull (5.79s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenScaleDownDelay (9.59s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenScaleDownOnAbort (8.90s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenToCanary (5.05s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenUpdate (5.92s) PASS test/e2e.TestFunctionalSuite/TestCanaryPromoteFull (9.92s) PASS test/e2e.TestFunctionalSuite/TestCompleteRolloutRestart (22.00s) PASS test/e2e.TestFunctionalSuite/TestContainerResourceFormats (3.07s) === RUN TestFunctionalSuite/TestControllerMetrics time="2024-10-30T14:34:24Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-10-30T14:34:24Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 6050 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc000bd9040, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc000ec3b00) /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc00122cc80?, 0xc0009faac8?, 0xc000e40f10?}, {0x3212d41, 0x4}, {0xc001297e70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc00122cc80?, 0xc0009faac8?, 0xc000ec3b00?}, {0xc001297e70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc000bd9040) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc000bd9040, 0xc000d5c1b0) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 5040 /usr/local/go/src/testing/testing.go:1648 +0x3ad --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.64s) FAIL test/e2e.TestFunctionalSuite/TestControllerMetrics (0.64s) PASS test/e2e.TestFunctionalSuite/TestFixInvalidSpec (21.84s) PASS test/e2e.TestFunctionalSuite/TestKubectlWaitForCompleted (23.92s) PASS test/e2e.TestFunctionalSuite/TestKubectlWaitForPaused (7.31s) PASS test/e2e.TestFunctionalSuite/TestMalformedRollout (3.46s) PASS test/e2e.TestFunctionalSuite/TestMalformedRolloutEphemeralCtr (3.01s) PASS test/e2e.TestFunctionalSuite/TestRolloutAbortRetryPromote (9.92s) PASS test/e2e.TestFunctionalSuite/TestRolloutBadToBadUpdate (22.95s) PASS test/e2e.TestFunctionalSuite/TestRolloutBadToGoodSetWeightUpdate (23.41s) PASS test/e2e.TestFunctionalSuite/TestRolloutBadToGoodUpdate (18.06s) PASS test/e2e.TestFunctionalSuite/TestRolloutGoodToBadUpdate (25.54s) PASS test/e2e.TestFunctionalSuite/TestRolloutPDBRestart (46.78s) PASS test/e2e.TestFunctionalSuite/TestRolloutPauseDurationGreaterThanProgressDeadlineSeconds (37.51s) PASS test/e2e.TestFunctionalSuite/TestRolloutPodVolumesItemsSpec (9.49s) PASS test/e2e.TestFunctionalSuite/TestRolloutRestart (18.05s) PASS test/e2e.TestFunctionalSuite/TestWorkloadRef (19.24s) PASS test/e2e.TestFunctionalSuite/TestWorkloadRefTemplate (2.89s) === RUN TestFunctionalSuite time="2024-10-30T14:32:55Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" --- FAIL: TestFunctionalSuite (384.60s) FAIL test/e2e.TestFunctionalSuite (384.60s) PASS test/e2e.TestHeaderRoutingSuite/TestIstioHostHeaderRoute (15.26s) PASS test/e2e.TestHeaderRoutingSuite (15.60s) PASS test/e2e.TestIstioSuite/TestIstioAbortUpdate (7.48s) PASS test/e2e.TestIstioSuite/TestIstioAbortUpdateDeleteAllCanaryPods (9.52s) PASS test/e2e.TestIstioSuite/TestIstioHostSplit (18.15s) PASS test/e2e.TestIstioSuite/TestIstioHostSplitExperimentStep (19.10s) PASS test/e2e.TestIstioSuite/TestIstioHostSplitOnlyTls (8.91s) PASS test/e2e.TestIstioSuite/TestIstioSubsetSplit (10.60s) PASS test/e2e.TestIstioSuite/TestIstioSubsetSplitExperimentStep (19.22s) PASS test/e2e.TestIstioSuite/TestIstioSubsetSplitSingleRoute (8.83s) PASS test/e2e.TestIstioSuite/TestIstioUpdateInMiddleZeroCanaryReplicas (9.05s) PASS test/e2e.TestIstioSuite (111.29s) PASS test/e2e.TestMirrorRouteSuite/TestIstioHostMirrorRoute (15.63s) PASS test/e2e.TestMirrorRouteSuite (16.00s) PASS test/e2e.TestRollbackSuite/TestRollbackAnalysisOutsideWindow (30.83s) PASS test/e2e.TestRollbackSuite/TestRollbackAnalysisWithinWindow (24.10s) PASS test/e2e.TestRollbackSuite (57.61s) FAIL test/e2e DONE 104 tests, 6 skipped, 8 failures in 2420.508s === RUN TestAPISIXSuite/TestAPISIXCanarySetHeaderStep time="2024-10-30T14:42:50Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:42:52Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-10-30T14:42:53Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-10-30T14:42:54Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route configured\n" rollout=rollout-apisix-canary time="2024-10-30T14:42:55Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:42:55Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-10-30T14:42:55Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:42:55Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (5.03s) FAIL test/e2e.TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 1) (5.03s) === RUN TestAPISIXSuite --- FAIL: TestAPISIXSuite (5.44s) FAIL test/e2e.TestAPISIXSuite (re-run 1) (5.44s) FAIL test/e2e === RUN TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress time="2024-10-30T14:43:06Z" level=info msg="Deleting e2e-test-name=TestALBExperimentStepNoSetWeightMultiIngress" time="2024-10-30T14:43:07Z" level=info msg="service/alb-rollout-root created\n" rollout=alb-rollout time="2024-10-30T14:43:08Z" level=info msg="service/alb-rollout-canary created\n" rollout=alb-rollout time="2024-10-30T14:43:09Z" level=info msg="service/alb-rollout-stable created\n" rollout=alb-rollout time="2024-10-30T14:43:10Z" level=info msg="Warning: annotation \"kubernetes.io/ingress.class\" is deprecated, please use 'spec.ingressClassName' instead\ningress.networking.k8s.io/alb-rollout-multi-ingress-1 created\n" rollout=alb-rollout time="2024-10-30T14:43:11Z" level=info msg="Warning: annotation \"kubernetes.io/ingress.class\" is deprecated, please use 'spec.ingressClassName' instead\ningress.networking.k8s.io/alb-rollout-multi-ingress-2 created\n" rollout=alb-rollout time="2024-10-30T14:43:12Z" level=info msg="rollout.argoproj.io/alb-rollout created\n" rollout=alb-rollout time="2024-10-30T14:43:12Z" level=info msg="Waiting for condition: status=Healthy" rollout=alb-rollout time="2024-10-30T14:43:13Z" level=info msg="Condition 'status=Healthy' met after 1s" rollout=alb-rollout time="2024-10-30T14:43:13Z" level=info msg="Experiment expectation 'experiment count == 0' met" rollout=alb-rollout time="2024-10-30T14:43:13Z" level=info msg="Updated rollout pod spec: 2024-10-30T14:43:13.936188765Z" rollout=alb-rollout time="2024-10-30T14:43:13Z" level=info msg="Sleeping 10s" rollout=alb-rollout require.go:72: Error Trace: /root/go/pkg/mod/github.com/tj/assert@v0.0.3/require.go:72 /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/aws_test.go:298 /tmp/tmp.SIcIuinRDW/argo-rollouts/test/fixtures/then.go:28 /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/aws_test.go:287 Error: Not equal: expected: "{\"Type\":\"forward\",\"ForwardConfig\":{\"TargetGroups\":[{\"ServiceName\":\"alb-rollout-canary\",\"ServicePort\":\"80\",\"Weight\":0},{\"ServiceName\":\"\",\"ServicePort\":\"80\",\"Weight\":20},{\"ServiceName\":\"\",\"ServicePort\":\"80\",\"Weight\":20},{\"ServiceName\":\"alb-rollout-stable\",\"ServicePort\":\"80\",\"Weight\":60}]}}" actual : "{\"Type\":\"forward\",\"ForwardConfig\":{\"TargetGroups\":[{\"ServiceName\":\"alb-rollout-canary\",\"ServicePort\":\"80\",\"Weight\":0},{\"ServicePort\":\"80\",\"Weight\":20},{\"ServicePort\":\"80\",\"Weight\":20},{\"ServiceName\":\"alb-rollout-stable\",\"ServicePort\":\"80\",\"Weight\":60}]}}" Diff: --- Expected +++ Actual @@ -1 +1 @@ -{"Type":"forward","ForwardConfig":{"TargetGroups":[{"ServiceName":"alb-rollout-canary","ServicePort":"80","Weight":0},{"ServiceName":"","ServicePort":"80","Weight":20},{"ServiceName":"","ServicePort":"80","Weight":20},{"ServiceName":"alb-rollout-stable","ServicePort":"80","Weight":60}]}} +{"Type":"forward","ForwardConfig":{"TargetGroups":[{"ServiceName":"alb-rollout-canary","ServicePort":"80","Weight":0},{"ServicePort":"80","Weight":20},{"ServicePort":"80","Weight":20},{"ServiceName":"alb-rollout-stable","ServicePort":"80","Weight":60}]}} Test: TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress Name: alb-rollout Namespace: argo-rollouts Status: ◌ Progressing Message: more replicas need to be updated Strategy: Canary Step: 0/1 SetWeight: 0 ActualWeight: 0 Images: nginx:1.19-alpine (stable, Σ:experiment-alb-canary, Σ:experiment-alb-stable) Replicas: Desired: 1 Current: 1 Updated: 0 Ready: 1 Available: 1 NAME KIND STATUS AGE INFO ⟳ alb-rollout Rollout ◌ Progressing 12s ├──# revision:2 │ ├──⧉ alb-rollout-6c6554945c ReplicaSet • ScaledDown 11s canary │ └──Σ alb-rollout-6c6554945c-2-0 Experiment ◌ Running 10s │ ├──⧉ alb-rollout-6c6554945c-2-0-experiment-alb-canary ReplicaSet ✔ Healthy 10s │ │ └──□ alb-rollout-6c6554945c-2-0-experiment-alb-canary-hb25b Pod ✔ Running 10s ready:1/1 │ └──⧉ alb-rollout-6c6554945c-2-0-experiment-alb-stable ReplicaSet ✔ Healthy 10s │ └──□ alb-rollout-6c6554945c-2-0-experiment-alb-stable-tlp29 Pod ✔ Running 10s ready:1/1 └──# revision:1 └──⧉ alb-rollout-d6f4fdcdc ReplicaSet ✔ Healthy 12s stable └──□ alb-rollout-d6f4fdcdc-zbcwc Pod ✔ Running 12s ready:1/1 --- apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: annotations: rollout.argoproj.io/revision: "2" creationTimestamp: "2024-10-30T14:43:12Z" generation: 3 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestALBExperimentStepNoSetWeightMultiIngress name: alb-rollout namespace: argo-rollouts resourceVersion: "10277604" uid: 42288294-19dd-42c5-b1fb-ce9720113e17 spec: replicas: 1 selector: matchLabels: app: alb-rollout strategy: canary: canaryService: alb-rollout-canary stableService: alb-rollout-stable steps: - experiment: duration: 15s templates: - metadata: {} name: experiment-alb-canary specRef: canary weight: 20 - metadata: {} name: experiment-alb-stable specRef: stable weight: 20 trafficRouting: alb: ingresses: - alb-rollout-multi-ingress-1 - alb-rollout-multi-ingress-2 rootService: alb-rollout-root servicePort: 80 template: metadata: annotations: update: "2024-10-30T14:43:13.936188765Z" creationTimestamp: null labels: app: alb-rollout spec: containers: - image: nginx:1.19-alpine name: alb-rollout ports: - containerPort: 80 name: http protocol: TCP resources: requests: cpu: 5m memory: 16Mi status: HPAReplicas: 1 availableReplicas: 1 blueGreen: {} canary: currentExperiment: alb-rollout-6c6554945c-2-0 weights: additional: - weight: 20 - weight: 20 canary: podTemplateHash: 6c6554945c serviceName: alb-rollout-canary weight: 0 stable: podTemplateHash: d6f4fdcdc serviceName: alb-rollout-stable weight: 60 conditions: - lastTransitionTime: "2024-10-30T14:43:13Z" lastUpdateTime: "2024-10-30T14:43:13Z" message: Rollout has minimum availability reason: AvailableReason status: "True" type: Available - lastTransitionTime: "2024-10-30T14:43:13Z" lastUpdateTime: "2024-10-30T14:43:13Z" message: Rollout is not healthy reason: RolloutHealthy status: "False" type: Healthy - lastTransitionTime: "2024-10-30T14:43:13Z" lastUpdateTime: "2024-10-30T14:43:13Z" message: RolloutCompleted reason: RolloutCompleted status: "False" type: Completed - lastTransitionTime: "2024-10-30T14:43:12Z" lastUpdateTime: "2024-10-30T14:43:14Z" message: Created new replica set "alb-rollout-6c6554945c" reason: NewReplicaSetCreated status: "True" type: Progressing currentPodHash: 6c6554945c currentStepHash: 558cc8bf4d currentStepIndex: 0 message: more replicas need to be updated observedGeneration: "3" phase: Progressing readyReplicas: 1 replicas: 1 selector: app=alb-rollout stableRS: d6f4fdcdc 2024-10-30T14:43:12Z Normal RolloutAddedToInformer Rollout resource added to informer: argo-rollouts/alb-rollout 2024-10-30T14:43:12Z Normal RolloutNotCompleted Rollout not completed, started update to revision 1 (d6f4fdcdc) 2024-10-30T14:43:12Z Normal RolloutUpdated Rollout updated to revision 1 2024-10-30T14:43:12Z Normal NewReplicaSetCreated Created ReplicaSet alb-rollout-d6f4fdcdc (revision 1) 2024-10-30T14:43:15Z (x2) Normal PatchingALBIngress Updating Ingress `alb-rollout-multi-ingress-1` to desiredWeight '0' 2024-10-30T14:43:15Z (x2) Normal PatchingALBIngress Updating Ingress `alb-rollout-multi-ingress-2` to desiredWeight '0' 2024-10-30T14:43:12Z (x2) Normal TrafficWeightUpdated Traffic weight updated to 0 2024-10-30T14:43:12Z Normal ScalingReplicaSet Scaled up ReplicaSet alb-rollout-d6f4fdcdc (revision 1) from 0 to 1 2024-10-30T14:43:12Z Normal RolloutCompleted Rollout completed update to revision 1 (d6f4fdcdc): Initial deploy 2024-10-30T14:43:14Z (x2) Normal TrafficWeightUpdated Traffic weight updated 2024-10-30T14:43:13Z Normal SwitchService Switched selector for service 'alb-rollout-stable' from '' to 'd6f4fdcdc' 2024-10-30T14:43:13Z Normal SwitchService Switched selector for service 'alb-rollout-canary' from '' to 'd6f4fdcdc' 2024-10-30T14:43:13Z Normal RolloutNotCompleted Rollout not completed, started update to revision 2 (6c6554945c) 2024-10-30T14:43:14Z Normal RolloutUpdated Rollout updated to revision 2 2024-10-30T14:43:14Z Normal NewReplicaSetCreated Created ReplicaSet alb-rollout-6c6554945c (revision 2) 2024-10-30T14:43:14Z Normal ExperimentCreated Created Experiment 'alb-rollout-6c6554945c-2-0' 2024-10-30T14:43:15Z Normal TrafficWeightUpdated Traffic weight updated additional: [{20 } {20 }] time="2024-10-30T14:43:24Z" level=info msg="Deleting e2e-test-name=TestALBExperimentStepNoSetWeightMultiIngress" time="2024-10-30T14:43:24Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress (18.30s) FAIL test/e2e.TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress (re-run 1) (18.30s) === RUN TestAWSSuite --- FAIL: TestAWSSuite (18.69s) FAIL test/e2e.TestAWSSuite (re-run 1) (18.69s) FAIL test/e2e === RUN TestExperimentSuite/TestExperimentWithDryRunMetrics time="2024-10-30T14:43:35Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:43:36Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-10-30T14:43:37Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-10-30T14:43:37Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-10-30T14:44:08Z" level=info msg="Condition 'phase=Successful' met after 30s" time="2024-10-30T14:44:08Z" level=info msg="Sleeping 3s" time="2024-10-30T14:44:11Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-10-30T14:44:11Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-10-30T14:44:11Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 34s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 34s delay:26s │ └──□ experiment-with-dry-run-baseline-9wfmw Pod ✔ Running 34s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ e0b19688-4b81-40e1-a00c-4f4e770d0f7d.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-10-30T14:43:37Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "10278102" uid: b39385ed-c1ee-4091-b7d4-0b77a60531ff spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-10-30T14:43:38Z" conditions: - lastTransitionTime: "2024-10-30T14:44:08Z" lastUpdateTime: "2024-10-30T14:44:08Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-10-30T14:44:08Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-10-30T14:43:37Z (x2) Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-10-30T14:43:37Z (x2) Normal ExperimentPending Experiment transitioned from -> Pending 2024-10-30T14:43:37Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-10-30T14:43:38Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-10-30T14:43:38Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-10-30T14:43:38Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-10-30T14:43:38Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-10-30T14:44:08Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-10-30T14:44:08Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-10-30T14:44:08Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-10-30T14:44:11Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:44:11Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (36.42s) FAIL test/e2e.TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 1) (36.42s) === RUN TestExperimentSuite --- FAIL: TestExperimentSuite (36.80s) FAIL test/e2e.TestExperimentSuite (re-run 1) (36.80s) FAIL test/e2e === RUN TestFunctionalSuite/TestControllerMetrics time="2024-10-30T14:44:23Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-10-30T14:44:23Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 100 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc000501860, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc000bc7d00) /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc0005a3e40?, 0xc000986228?, 0xc000a0d4f0?}, {0x3212d41, 0x4}, {0xc0007dde70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc0005a3e40?, 0xc000986228?, 0xc000bc7d00?}, {0xc0007dde70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc000501860) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc000501860, 0xc0008d9200) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 15 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-10-30T14:44:23Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.53s) FAIL test/e2e.TestFunctionalSuite/TestControllerMetrics (re-run 1) (0.53s) === RUN TestFunctionalSuite time="2024-10-30T14:44:23Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" --- FAIL: TestFunctionalSuite (1.99s) FAIL test/e2e.TestFunctionalSuite (re-run 1) (1.99s) FAIL test/e2e DONE 2 runs, 112 tests, 6 skipped, 16 failures in 2524.565s === RUN TestAPISIXSuite/TestAPISIXCanarySetHeaderStep time="2024-10-30T14:44:33Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:44:35Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-10-30T14:44:36Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-10-30T14:44:37Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:44:38Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:44:38Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-10-30T14:44:38Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:44:38Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (4.66s) FAIL test/e2e.TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 2) (4.66s) === RUN TestAPISIXSuite --- FAIL: TestAPISIXSuite (5.09s) FAIL test/e2e.TestAPISIXSuite (re-run 2) (5.09s) FAIL test/e2e PASS test/e2e.TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress (re-run 2) (22.76s) PASS test/e2e.TestAWSSuite (re-run 2) (23.15s) PASS test/e2e === RUN TestExperimentSuite/TestExperimentWithDryRunMetrics time="2024-10-30T14:45:21Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:45:23Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-10-30T14:45:24Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-10-30T14:45:24Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-10-30T14:45:56Z" level=info msg="Condition 'phase=Successful' met after 31s" time="2024-10-30T14:45:56Z" level=info msg="Sleeping 3s" time="2024-10-30T14:45:59Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-10-30T14:45:59Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-10-30T14:45:59Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 35s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 35s delay:26s │ └──□ experiment-with-dry-run-baseline-s9h9p Pod ✔ Running 35s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ 61d7421f-1e4e-4f4f-928b-08511edb1a20.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-10-30T14:45:24Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "10279234" uid: 95d9950d-2c96-4ae1-88a6-465495232909 spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-10-30T14:45:26Z" conditions: - lastTransitionTime: "2024-10-30T14:45:56Z" lastUpdateTime: "2024-10-30T14:45:56Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-10-30T14:45:56Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-10-30T14:45:24Z (x2) Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-10-30T14:45:24Z (x2) Normal ExperimentPending Experiment transitioned from -> Pending 2024-10-30T14:45:24Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-10-30T14:45:26Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-10-30T14:45:26Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-10-30T14:45:26Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-10-30T14:45:26Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-10-30T14:45:56Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-10-30T14:45:56Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-10-30T14:45:56Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-10-30T14:45:59Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:45:59Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (37.58s) FAIL test/e2e.TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 2) (37.58s) === RUN TestExperimentSuite --- FAIL: TestExperimentSuite (37.97s) FAIL test/e2e.TestExperimentSuite (re-run 2) (37.97s) FAIL test/e2e === RUN TestFunctionalSuite/TestControllerMetrics time="2024-10-30T14:46:11Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-10-30T14:46:11Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 114 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc000aae340, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc0006b3f00) /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc0007914c0?, 0xc0001ab5f8?, 0xc00087d650?}, {0x3212d41, 0x4}, {0xc000b83e70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc0007914c0?, 0xc0001ab5f8?, 0xc0006b3f00?}, {0xc000b83e70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc000aae340) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc000aae340, 0xc00087eab0) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 83 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-10-30T14:46:11Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.81s) FAIL test/e2e.TestFunctionalSuite/TestControllerMetrics (re-run 2) (0.81s) === RUN TestFunctionalSuite time="2024-10-30T14:46:11Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" --- FAIL: TestFunctionalSuite (2.28s) FAIL test/e2e.TestFunctionalSuite (re-run 2) (2.28s) FAIL test/e2e DONE 3 runs, 120 tests, 6 skipped, 22 failures in 2632.809s === RUN TestAPISIXSuite/TestAPISIXCanarySetHeaderStep time="2024-10-30T14:46:22Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:46:23Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-10-30T14:46:24Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-10-30T14:46:25Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:46:26Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:46:26Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-10-30T14:46:26Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:46:27Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (4.93s) FAIL test/e2e.TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 3) (4.93s) === RUN TestAPISIXSuite --- FAIL: TestAPISIXSuite (5.30s) FAIL test/e2e.TestAPISIXSuite (re-run 3) (5.30s) FAIL test/e2e === RUN TestExperimentSuite/TestExperimentWithDryRunMetrics time="2024-10-30T14:46:37Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:46:38Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-10-30T14:46:39Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-10-30T14:46:39Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-10-30T14:47:11Z" level=info msg="Condition 'phase=Successful' met after 31s" time="2024-10-30T14:47:11Z" level=info msg="Sleeping 3s" time="2024-10-30T14:47:14Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-10-30T14:47:14Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-10-30T14:47:14Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 35s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 35s delay:26s │ └──□ experiment-with-dry-run-baseline-g6ll6 Pod ✔ Running 35s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ 69a022cc-dc98-48cc-8686-d40fdd4763f7.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-10-30T14:46:39Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "10279817" uid: d506ad7e-6feb-46bb-b266-97f09d2e3d1a spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-10-30T14:46:41Z" conditions: - lastTransitionTime: "2024-10-30T14:47:11Z" lastUpdateTime: "2024-10-30T14:47:11Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-10-30T14:47:11Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-10-30T14:46:39Z (x2) Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-10-30T14:46:39Z (x2) Normal ExperimentPending Experiment transitioned from -> Pending 2024-10-30T14:46:39Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-10-30T14:46:41Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-10-30T14:46:41Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-10-30T14:46:41Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-10-30T14:46:41Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-10-30T14:47:11Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-10-30T14:47:11Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-10-30T14:47:11Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-10-30T14:47:14Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:47:14Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (37.23s) FAIL test/e2e.TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 3) (37.23s) === RUN TestExperimentSuite --- FAIL: TestExperimentSuite (37.59s) FAIL test/e2e.TestExperimentSuite (re-run 3) (37.59s) FAIL test/e2e === RUN TestFunctionalSuite/TestControllerMetrics time="2024-10-30T14:47:25Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-10-30T14:47:26Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 54 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc0007829c0, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc000c04000) /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc000c0d100?, 0xc000c09728?, 0xc000cbda80?}, {0x3212d41, 0x4}, {0xc0007bbe70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc000c0d100?, 0xc000c09728?, 0xc000c04000?}, {0xc0007bbe70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc0007829c0) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc0007829c0, 0xc0004e4bd0) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 37 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-10-30T14:47:26Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.65s) FAIL test/e2e.TestFunctionalSuite/TestControllerMetrics (re-run 3) (0.65s) === RUN TestFunctionalSuite time="2024-10-30T14:47:25Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" --- FAIL: TestFunctionalSuite (2.27s) FAIL test/e2e.TestFunctionalSuite (re-run 3) (2.27s) FAIL test/e2e DONE 4 runs, 126 tests, 6 skipped, 28 failures in 2707.610s === RUN TestAPISIXSuite/TestAPISIXCanarySetHeaderStep time="2024-10-30T14:47:36Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:47:38Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-10-30T14:47:39Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-10-30T14:47:40Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:47:41Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:47:41Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-10-30T14:47:41Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:47:41Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (4.99s) FAIL test/e2e.TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 4) (4.99s) === RUN TestAPISIXSuite --- FAIL: TestAPISIXSuite (5.39s) FAIL test/e2e.TestAPISIXSuite (re-run 4) (5.39s) FAIL test/e2e === RUN TestExperimentSuite/TestExperimentWithDryRunMetrics time="2024-10-30T14:47:52Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:47:53Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-10-30T14:47:54Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-10-30T14:47:54Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-10-30T14:48:25Z" level=info msg="Condition 'phase=Successful' met after 30s" time="2024-10-30T14:48:25Z" level=info msg="Sleeping 3s" time="2024-10-30T14:48:28Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-10-30T14:48:28Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-10-30T14:48:28Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 34s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 34s delay:26s │ └──□ experiment-with-dry-run-baseline-xrffd Pod ✔ Running 34s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ b68d9d01-d22a-4ef8-b1ff-3ae24ca5763d.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-10-30T14:47:54Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "10280805" uid: 75b788be-0f28-4a04-a13a-747befb476f9 spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-10-30T14:47:55Z" conditions: - lastTransitionTime: "2024-10-30T14:48:25Z" lastUpdateTime: "2024-10-30T14:48:25Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-10-30T14:48:25Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-10-30T14:47:54Z (x2) Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-10-30T14:47:54Z (x2) Normal ExperimentPending Experiment transitioned from -> Pending 2024-10-30T14:47:54Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-10-30T14:47:55Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-10-30T14:47:55Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-10-30T14:47:55Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-10-30T14:47:55Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-10-30T14:48:25Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-10-30T14:48:25Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-10-30T14:48:25Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-10-30T14:48:28Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:48:28Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (36.31s) FAIL test/e2e.TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 4) (36.31s) === RUN TestExperimentSuite --- FAIL: TestExperimentSuite (36.68s) FAIL test/e2e.TestExperimentSuite (re-run 4) (36.68s) FAIL test/e2e === RUN TestFunctionalSuite/TestControllerMetrics time="2024-10-30T14:48:40Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-10-30T14:48:40Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 89 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc000102ea0, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc000584f00) /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc000b07b00?, 0xc000a34168?, 0xc000b402d0?}, {0x3212d41, 0x4}, {0xc00087be70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc000b07b00?, 0xc000a34168?, 0xc000584f00?}, {0xc00087be70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc000102ea0) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc000102ea0, 0xc000ad7440) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 15 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-10-30T14:48:40Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.61s) FAIL test/e2e.TestFunctionalSuite/TestControllerMetrics (re-run 4) (0.61s) === RUN TestFunctionalSuite time="2024-10-30T14:48:40Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" --- FAIL: TestFunctionalSuite (2.09s) FAIL test/e2e.TestFunctionalSuite (re-run 4) (2.09s) FAIL test/e2e DONE 5 runs, 132 tests, 6 skipped, 34 failures in 2781.624s === RUN TestAPISIXSuite/TestAPISIXCanarySetHeaderStep time="2024-10-30T14:48:50Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:48:52Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-10-30T14:48:53Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-10-30T14:48:54Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:48:55Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:48:55Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-10-30T14:48:55Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:48:55Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (4.70s) FAIL test/e2e.TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 5) (4.70s) === RUN TestAPISIXSuite --- FAIL: TestAPISIXSuite (5.03s) FAIL test/e2e.TestAPISIXSuite (re-run 5) (5.03s) FAIL test/e2e === RUN TestExperimentSuite/TestExperimentWithDryRunMetrics time="2024-10-30T14:49:05Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:49:07Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-10-30T14:49:08Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-10-30T14:49:08Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-10-30T14:49:39Z" level=info msg="Condition 'phase=Successful' met after 30s" time="2024-10-30T14:49:39Z" level=info msg="Sleeping 3s" time="2024-10-30T14:49:42Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-10-30T14:49:42Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-10-30T14:49:42Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 34s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 34s delay:26s │ └──□ experiment-with-dry-run-baseline-f6nkh Pod ✔ Running 34s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ 8b04005c-0639-4900-81dd-22ea7dc1e63a.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-10-30T14:49:08Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "10281497" uid: 3c4d8d24-8c1a-4a78-99bf-f046ca53b8ac spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-10-30T14:49:09Z" conditions: - lastTransitionTime: "2024-10-30T14:49:39Z" lastUpdateTime: "2024-10-30T14:49:39Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-10-30T14:49:39Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-10-30T14:49:08Z (x2) Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-10-30T14:49:08Z (x2) Normal ExperimentPending Experiment transitioned from -> Pending 2024-10-30T14:49:08Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-10-30T14:49:09Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-10-30T14:49:09Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-10-30T14:49:09Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-10-30T14:49:09Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-10-30T14:49:39Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-10-30T14:49:39Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-10-30T14:49:39Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-10-30T14:49:42Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:49:42Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (36.77s) FAIL test/e2e.TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 5) (36.77s) === RUN TestExperimentSuite --- FAIL: TestExperimentSuite (37.15s) FAIL test/e2e.TestExperimentSuite (re-run 5) (37.15s) FAIL test/e2e === RUN TestFunctionalSuite/TestControllerMetrics time="2024-10-30T14:49:54Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-10-30T14:49:54Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 30 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc000502d00, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc0001cdc00) /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc000a078c0?, 0xc000992170?, 0xc000741de0?}, {0x3212d41, 0x4}, {0xc000a91e70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc000a078c0?, 0xc000992170?, 0xc0001cdc00?}, {0xc000a91e70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc000502d00) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc000502d00, 0xc00003f5f0) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 83 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-10-30T14:49:54Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.71s) FAIL test/e2e.TestFunctionalSuite/TestControllerMetrics (re-run 5) (0.71s) === RUN TestFunctionalSuite time="2024-10-30T14:49:54Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" --- FAIL: TestFunctionalSuite (2.21s) FAIL test/e2e.TestFunctionalSuite (re-run 5) (2.21s) FAIL test/e2e === Skipped === SKIP: test/e2e TestAppMeshSuite (0.05s) === SKIP: test/e2e TestAWSSuite/TestALBBlueGreenUpdate (0.59s) time="2024-10-30T14:19:08Z" level=info msg="Deleting e2e-test-name=TestALBBlueGreenUpdate" time="2024-10-30T14:19:09Z" level=info msg="Deleting e2e-test-name=TestALBBlueGreenUpdate" --- SKIP: TestAWSSuite/TestALBBlueGreenUpdate (0.59s) === SKIP: test/e2e TestAWSSuite/TestALBCanaryUpdate (0.59s) time="2024-10-30T14:19:09Z" level=info msg="Deleting e2e-test-name=TestALBCanaryUpdate" time="2024-10-30T14:19:09Z" level=info msg="Deleting e2e-test-name=TestALBCanaryUpdate" --- SKIP: TestAWSSuite/TestALBCanaryUpdate (0.59s) === SKIP: test/e2e TestAWSSuite/TestALBCanaryUpdateMultiIngress (0.58s) time="2024-10-30T14:19:10Z" level=info msg="Deleting e2e-test-name=TestALBCanaryUpdateMultiIngress" time="2024-10-30T14:19:10Z" level=info msg="Deleting e2e-test-name=TestALBCanaryUpdateMultiIngress" --- SKIP: TestAWSSuite/TestALBCanaryUpdateMultiIngress (0.58s) === SKIP: test/e2e TestSMIIngressSuite (0.05s) === SKIP: test/e2e TestSMISuite (0.05s) === Failed === FAIL: test/e2e TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (7.60s) time="2024-10-30T14:18:32Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:18:34Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-10-30T14:18:36Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-10-30T14:18:37Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route created\n" rollout=rollout-apisix-canary time="2024-10-30T14:18:39Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:18:39Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-10-30T14:18:39Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (7.60s) === FAIL: test/e2e TestAPISIXSuite (36.31s) === FAIL: test/e2e TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress (18.48s) time="2024-10-30T14:20:24Z" level=info msg="Deleting e2e-test-name=TestALBExperimentStepNoSetWeightMultiIngress" time="2024-10-30T14:20:26Z" level=info msg="service/alb-rollout-root created\n" rollout=alb-rollout time="2024-10-30T14:20:27Z" level=info msg="service/alb-rollout-canary created\n" rollout=alb-rollout time="2024-10-30T14:20:28Z" level=info msg="service/alb-rollout-stable created\n" rollout=alb-rollout time="2024-10-30T14:20:29Z" level=info msg="Warning: annotation \"kubernetes.io/ingress.class\" is deprecated, please use 'spec.ingressClassName' instead\ningress.networking.k8s.io/alb-rollout-multi-ingress-1 created\n" rollout=alb-rollout time="2024-10-30T14:20:30Z" level=info msg="Warning: annotation \"kubernetes.io/ingress.class\" is deprecated, please use 'spec.ingressClassName' instead\ningress.networking.k8s.io/alb-rollout-multi-ingress-2 created\n" rollout=alb-rollout time="2024-10-30T14:20:31Z" level=info msg="rollout.argoproj.io/alb-rollout created\n" rollout=alb-rollout time="2024-10-30T14:20:31Z" level=info msg="Waiting for condition: status=Healthy" rollout=alb-rollout time="2024-10-30T14:20:32Z" level=info msg="Condition 'status=Healthy' met after 0s" rollout=alb-rollout time="2024-10-30T14:20:32Z" level=info msg="Experiment expectation 'experiment count == 0' met" rollout=alb-rollout time="2024-10-30T14:20:32Z" level=info msg="Updated rollout pod spec: 2024-10-30T14:20:32.566524227Z" rollout=alb-rollout time="2024-10-30T14:20:32Z" level=info msg="Sleeping 10s" rollout=alb-rollout require.go:72: Error Trace: /root/go/pkg/mod/github.com/tj/assert@v0.0.3/require.go:72 /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/aws_test.go:298 /tmp/tmp.SIcIuinRDW/argo-rollouts/test/fixtures/then.go:28 /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/aws_test.go:287 Error: Not equal: expected: "{\"Type\":\"forward\",\"ForwardConfig\":{\"TargetGroups\":[{\"ServiceName\":\"alb-rollout-canary\",\"ServicePort\":\"80\",\"Weight\":0},{\"ServiceName\":\"\",\"ServicePort\":\"80\",\"Weight\":20},{\"ServiceName\":\"\",\"ServicePort\":\"80\",\"Weight\":20},{\"ServiceName\":\"alb-rollout-stable\",\"ServicePort\":\"80\",\"Weight\":60}]}}" actual : "{\"Type\":\"forward\",\"ForwardConfig\":{\"TargetGroups\":[{\"ServiceName\":\"alb-rollout-canary\",\"ServicePort\":\"80\",\"Weight\":0},{\"ServicePort\":\"80\",\"Weight\":20},{\"ServicePort\":\"80\",\"Weight\":20},{\"ServiceName\":\"alb-rollout-stable\",\"ServicePort\":\"80\",\"Weight\":60}]}}" Diff: --- Expected +++ Actual @@ -1 +1 @@ -{"Type":"forward","ForwardConfig":{"TargetGroups":[{"ServiceName":"alb-rollout-canary","ServicePort":"80","Weight":0},{"ServiceName":"","ServicePort":"80","Weight":20},{"ServiceName":"","ServicePort":"80","Weight":20},{"ServiceName":"alb-rollout-stable","ServicePort":"80","Weight":60}]}} +{"Type":"forward","ForwardConfig":{"TargetGroups":[{"ServiceName":"alb-rollout-canary","ServicePort":"80","Weight":0},{"ServicePort":"80","Weight":20},{"ServicePort":"80","Weight":20},{"ServiceName":"alb-rollout-stable","ServicePort":"80","Weight":60}]}} Test: TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress Name: alb-rollout Namespace: argo-rollouts Status: ◌ Progressing Message: more replicas need to be updated Strategy: Canary Step: 0/1 SetWeight: 0 ActualWeight: 0 Images: nginx:1.19-alpine (stable, Σ:experiment-alb-canary, Σ:experiment-alb-stable) Replicas: Desired: 1 Current: 1 Updated: 0 Ready: 1 Available: 1 NAME KIND STATUS AGE INFO ⟳ alb-rollout Rollout ◌ Progressing 11s ├──# revision:2 │ ├──⧉ alb-rollout-6bc4748d8c ReplicaSet • ScaledDown 10s canary │ └──Σ alb-rollout-6bc4748d8c-2-0 Experiment ◌ Running 10s │ ├──⧉ alb-rollout-6bc4748d8c-2-0-experiment-alb-canary ReplicaSet ✔ Healthy 10s │ │ └──□ alb-rollout-6bc4748d8c-2-0-experiment-alb-canary-46cnd Pod ✔ Running 10s ready:1/1 │ └──⧉ alb-rollout-6bc4748d8c-2-0-experiment-alb-stable ReplicaSet ✔ Healthy 10s │ └──□ alb-rollout-6bc4748d8c-2-0-experiment-alb-stable-gzvk9 Pod ✔ Running 10s ready:1/1 └──# revision:1 └──⧉ alb-rollout-d6f4fdcdc ReplicaSet ✔ Healthy 11s stable └──□ alb-rollout-d6f4fdcdc-glgpk Pod ✔ Running 11s ready:1/1 --- apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: annotations: rollout.argoproj.io/revision: "2" creationTimestamp: "2024-10-30T14:20:31Z" generation: 3 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestALBExperimentStepNoSetWeightMultiIngress name: alb-rollout namespace: argo-rollouts resourceVersion: "10254495" uid: d47c7a56-cd3c-4b0e-a157-8f63888ebd57 spec: replicas: 1 selector: matchLabels: app: alb-rollout strategy: canary: canaryService: alb-rollout-canary stableService: alb-rollout-stable steps: - experiment: duration: 15s templates: - metadata: {} name: experiment-alb-canary specRef: canary weight: 20 - metadata: {} name: experiment-alb-stable specRef: stable weight: 20 trafficRouting: alb: ingresses: - alb-rollout-multi-ingress-1 - alb-rollout-multi-ingress-2 rootService: alb-rollout-root servicePort: 80 template: metadata: annotations: update: "2024-10-30T14:20:32.566524227Z" creationTimestamp: null labels: app: alb-rollout spec: containers: - image: nginx:1.19-alpine name: alb-rollout ports: - containerPort: 80 name: http protocol: TCP resources: requests: cpu: 5m memory: 16Mi status: HPAReplicas: 1 availableReplicas: 1 blueGreen: {} canary: currentExperiment: alb-rollout-6bc4748d8c-2-0 weights: additional: - weight: 20 - weight: 20 canary: podTemplateHash: 6bc4748d8c serviceName: alb-rollout-canary weight: 0 stable: podTemplateHash: d6f4fdcdc serviceName: alb-rollout-stable weight: 60 conditions: - lastTransitionTime: "2024-10-30T14:20:32Z" lastUpdateTime: "2024-10-30T14:20:32Z" message: Rollout has minimum availability reason: AvailableReason status: "True" type: Available - lastTransitionTime: "2024-10-30T14:20:32Z" lastUpdateTime: "2024-10-30T14:20:32Z" message: Rollout is not healthy reason: RolloutHealthy status: "False" type: Healthy - lastTransitionTime: "2024-10-30T14:20:32Z" lastUpdateTime: "2024-10-30T14:20:32Z" message: RolloutCompleted reason: RolloutCompleted status: "False" type: Completed - lastTransitionTime: "2024-10-30T14:20:31Z" lastUpdateTime: "2024-10-30T14:20:32Z" message: Created new replica set "alb-rollout-6bc4748d8c" reason: NewReplicaSetCreated status: "True" type: Progressing currentPodHash: 6bc4748d8c currentStepHash: 558cc8bf4d currentStepIndex: 0 message: more replicas need to be updated observedGeneration: "3" phase: Progressing readyReplicas: 1 replicas: 1 selector: app=alb-rollout stableRS: d6f4fdcdc 2024-10-30T14:20:31Z Normal RolloutAddedToInformer Rollout resource added to informer: argo-rollouts/alb-rollout 2024-10-30T14:20:31Z Normal RolloutNotCompleted Rollout not completed, started update to revision 1 (d6f4fdcdc) 2024-10-30T14:20:31Z Normal RolloutUpdated Rollout updated to revision 1 2024-10-30T14:20:31Z Normal NewReplicaSetCreated Created ReplicaSet alb-rollout-d6f4fdcdc (revision 1) 2024-10-30T14:20:34Z (x2) Normal PatchingALBIngress Updating Ingress `alb-rollout-multi-ingress-1` to desiredWeight '0' 2024-10-30T14:20:34Z (x2) Normal PatchingALBIngress Updating Ingress `alb-rollout-multi-ingress-2` to desiredWeight '0' 2024-10-30T14:20:31Z (x2) Normal TrafficWeightUpdated Traffic weight updated to 0 2024-10-30T14:20:31Z Normal ScalingReplicaSet Scaled up ReplicaSet alb-rollout-d6f4fdcdc (revision 1) from 0 to 1 2024-10-30T14:20:31Z Normal RolloutCompleted Rollout completed update to revision 1 (d6f4fdcdc): Initial deploy 2024-10-30T14:20:32Z (x2) Normal TrafficWeightUpdated Traffic weight updated 2024-10-30T14:20:32Z Normal SwitchService Switched selector for service 'alb-rollout-stable' from '' to 'd6f4fdcdc' 2024-10-30T14:20:32Z Normal SwitchService Switched selector for service 'alb-rollout-canary' from '' to 'd6f4fdcdc' 2024-10-30T14:20:32Z Normal RolloutNotCompleted Rollout not completed, started update to revision 2 (6bc4748d8c) 2024-10-30T14:20:32Z Normal RolloutUpdated Rollout updated to revision 2 2024-10-30T14:20:32Z Normal NewReplicaSetCreated Created ReplicaSet alb-rollout-6bc4748d8c (revision 2) 2024-10-30T14:20:32Z Normal ExperimentCreated Created Experiment 'alb-rollout-6bc4748d8c-2-0' 2024-10-30T14:20:34Z Normal TrafficWeightUpdated Traffic weight updated additional: [{20 } {20 }] time="2024-10-30T14:20:42Z" level=info msg="Deleting e2e-test-name=TestALBExperimentStepNoSetWeightMultiIngress" --- FAIL: TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress (18.48s) === FAIL: test/e2e TestAWSSuite (163.89s) === FAIL: test/e2e TestExperimentSuite/TestExperimentWithDryRunMetrics (36.79s) time="2024-10-30T14:30:36Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:30:38Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-10-30T14:30:39Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-10-30T14:30:39Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-10-30T14:31:10Z" level=info msg="Condition 'phase=Successful' met after 30s" time="2024-10-30T14:31:10Z" level=info msg="Sleeping 3s" time="2024-10-30T14:31:13Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-10-30T14:31:13Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-10-30T14:31:13Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 34s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 34s delay:26s │ └──□ experiment-with-dry-run-baseline-r8p2q Pod ✔ Running 34s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ 403c550f-f53b-4b1b-a0a9-3af4f099a7a8.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-10-30T14:30:39Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "10264901" uid: 0e58c5a1-ab85-4241-abd0-0e7a86e7ed4c spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-10-30T14:30:40Z" conditions: - lastTransitionTime: "2024-10-30T14:31:10Z" lastUpdateTime: "2024-10-30T14:31:10Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-10-30T14:31:10Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-10-30T14:30:39Z Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-10-30T14:30:39Z Normal ExperimentPending Experiment transitioned from -> Pending 2024-10-30T14:30:39Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-10-30T14:30:40Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-10-30T14:30:40Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-10-30T14:30:40Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-10-30T14:30:40Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-10-30T14:31:10Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-10-30T14:31:10Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-10-30T14:31:10Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-10-30T14:31:13Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (36.79s) === FAIL: test/e2e TestExperimentSuite (127.36s) === FAIL: test/e2e TestFunctionalSuite/TestControllerMetrics (0.64s) time="2024-10-30T14:34:24Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-10-30T14:34:24Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 6050 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc000bd9040, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc000ec3b00) /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc00122cc80?, 0xc0009faac8?, 0xc000e40f10?}, {0x3212d41, 0x4}, {0xc001297e70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc00122cc80?, 0xc0009faac8?, 0xc000ec3b00?}, {0xc001297e70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc000bd9040) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc000bd9040, 0xc000d5c1b0) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 5040 /usr/local/go/src/testing/testing.go:1648 +0x3ad --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.64s) === FAIL: test/e2e TestFunctionalSuite (384.60s) time="2024-10-30T14:32:55Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" === FAIL: test/e2e TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 1) (5.03s) time="2024-10-30T14:42:50Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:42:52Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-10-30T14:42:53Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-10-30T14:42:54Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route configured\n" rollout=rollout-apisix-canary time="2024-10-30T14:42:55Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:42:55Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-10-30T14:42:55Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:42:55Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (5.03s) === FAIL: test/e2e TestAPISIXSuite (re-run 1) (5.44s) === FAIL: test/e2e TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress (re-run 1) (18.30s) time="2024-10-30T14:43:06Z" level=info msg="Deleting e2e-test-name=TestALBExperimentStepNoSetWeightMultiIngress" time="2024-10-30T14:43:07Z" level=info msg="service/alb-rollout-root created\n" rollout=alb-rollout time="2024-10-30T14:43:08Z" level=info msg="service/alb-rollout-canary created\n" rollout=alb-rollout time="2024-10-30T14:43:09Z" level=info msg="service/alb-rollout-stable created\n" rollout=alb-rollout time="2024-10-30T14:43:10Z" level=info msg="Warning: annotation \"kubernetes.io/ingress.class\" is deprecated, please use 'spec.ingressClassName' instead\ningress.networking.k8s.io/alb-rollout-multi-ingress-1 created\n" rollout=alb-rollout time="2024-10-30T14:43:11Z" level=info msg="Warning: annotation \"kubernetes.io/ingress.class\" is deprecated, please use 'spec.ingressClassName' instead\ningress.networking.k8s.io/alb-rollout-multi-ingress-2 created\n" rollout=alb-rollout time="2024-10-30T14:43:12Z" level=info msg="rollout.argoproj.io/alb-rollout created\n" rollout=alb-rollout time="2024-10-30T14:43:12Z" level=info msg="Waiting for condition: status=Healthy" rollout=alb-rollout time="2024-10-30T14:43:13Z" level=info msg="Condition 'status=Healthy' met after 1s" rollout=alb-rollout time="2024-10-30T14:43:13Z" level=info msg="Experiment expectation 'experiment count == 0' met" rollout=alb-rollout time="2024-10-30T14:43:13Z" level=info msg="Updated rollout pod spec: 2024-10-30T14:43:13.936188765Z" rollout=alb-rollout time="2024-10-30T14:43:13Z" level=info msg="Sleeping 10s" rollout=alb-rollout require.go:72: Error Trace: /root/go/pkg/mod/github.com/tj/assert@v0.0.3/require.go:72 /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/aws_test.go:298 /tmp/tmp.SIcIuinRDW/argo-rollouts/test/fixtures/then.go:28 /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/aws_test.go:287 Error: Not equal: expected: "{\"Type\":\"forward\",\"ForwardConfig\":{\"TargetGroups\":[{\"ServiceName\":\"alb-rollout-canary\",\"ServicePort\":\"80\",\"Weight\":0},{\"ServiceName\":\"\",\"ServicePort\":\"80\",\"Weight\":20},{\"ServiceName\":\"\",\"ServicePort\":\"80\",\"Weight\":20},{\"ServiceName\":\"alb-rollout-stable\",\"ServicePort\":\"80\",\"Weight\":60}]}}" actual : "{\"Type\":\"forward\",\"ForwardConfig\":{\"TargetGroups\":[{\"ServiceName\":\"alb-rollout-canary\",\"ServicePort\":\"80\",\"Weight\":0},{\"ServicePort\":\"80\",\"Weight\":20},{\"ServicePort\":\"80\",\"Weight\":20},{\"ServiceName\":\"alb-rollout-stable\",\"ServicePort\":\"80\",\"Weight\":60}]}}" Diff: --- Expected +++ Actual @@ -1 +1 @@ -{"Type":"forward","ForwardConfig":{"TargetGroups":[{"ServiceName":"alb-rollout-canary","ServicePort":"80","Weight":0},{"ServiceName":"","ServicePort":"80","Weight":20},{"ServiceName":"","ServicePort":"80","Weight":20},{"ServiceName":"alb-rollout-stable","ServicePort":"80","Weight":60}]}} +{"Type":"forward","ForwardConfig":{"TargetGroups":[{"ServiceName":"alb-rollout-canary","ServicePort":"80","Weight":0},{"ServicePort":"80","Weight":20},{"ServicePort":"80","Weight":20},{"ServiceName":"alb-rollout-stable","ServicePort":"80","Weight":60}]}} Test: TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress Name: alb-rollout Namespace: argo-rollouts Status: ◌ Progressing Message: more replicas need to be updated Strategy: Canary Step: 0/1 SetWeight: 0 ActualWeight: 0 Images: nginx:1.19-alpine (stable, Σ:experiment-alb-canary, Σ:experiment-alb-stable) Replicas: Desired: 1 Current: 1 Updated: 0 Ready: 1 Available: 1 NAME KIND STATUS AGE INFO ⟳ alb-rollout Rollout ◌ Progressing 12s ├──# revision:2 │ ├──⧉ alb-rollout-6c6554945c ReplicaSet • ScaledDown 11s canary │ └──Σ alb-rollout-6c6554945c-2-0 Experiment ◌ Running 10s │ ├──⧉ alb-rollout-6c6554945c-2-0-experiment-alb-canary ReplicaSet ✔ Healthy 10s │ │ └──□ alb-rollout-6c6554945c-2-0-experiment-alb-canary-hb25b Pod ✔ Running 10s ready:1/1 │ └──⧉ alb-rollout-6c6554945c-2-0-experiment-alb-stable ReplicaSet ✔ Healthy 10s │ └──□ alb-rollout-6c6554945c-2-0-experiment-alb-stable-tlp29 Pod ✔ Running 10s ready:1/1 └──# revision:1 └──⧉ alb-rollout-d6f4fdcdc ReplicaSet ✔ Healthy 12s stable └──□ alb-rollout-d6f4fdcdc-zbcwc Pod ✔ Running 12s ready:1/1 --- apiVersion: argoproj.io/v1alpha1 kind: Rollout metadata: annotations: rollout.argoproj.io/revision: "2" creationTimestamp: "2024-10-30T14:43:12Z" generation: 3 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestALBExperimentStepNoSetWeightMultiIngress name: alb-rollout namespace: argo-rollouts resourceVersion: "10277604" uid: 42288294-19dd-42c5-b1fb-ce9720113e17 spec: replicas: 1 selector: matchLabels: app: alb-rollout strategy: canary: canaryService: alb-rollout-canary stableService: alb-rollout-stable steps: - experiment: duration: 15s templates: - metadata: {} name: experiment-alb-canary specRef: canary weight: 20 - metadata: {} name: experiment-alb-stable specRef: stable weight: 20 trafficRouting: alb: ingresses: - alb-rollout-multi-ingress-1 - alb-rollout-multi-ingress-2 rootService: alb-rollout-root servicePort: 80 template: metadata: annotations: update: "2024-10-30T14:43:13.936188765Z" creationTimestamp: null labels: app: alb-rollout spec: containers: - image: nginx:1.19-alpine name: alb-rollout ports: - containerPort: 80 name: http protocol: TCP resources: requests: cpu: 5m memory: 16Mi status: HPAReplicas: 1 availableReplicas: 1 blueGreen: {} canary: currentExperiment: alb-rollout-6c6554945c-2-0 weights: additional: - weight: 20 - weight: 20 canary: podTemplateHash: 6c6554945c serviceName: alb-rollout-canary weight: 0 stable: podTemplateHash: d6f4fdcdc serviceName: alb-rollout-stable weight: 60 conditions: - lastTransitionTime: "2024-10-30T14:43:13Z" lastUpdateTime: "2024-10-30T14:43:13Z" message: Rollout has minimum availability reason: AvailableReason status: "True" type: Available - lastTransitionTime: "2024-10-30T14:43:13Z" lastUpdateTime: "2024-10-30T14:43:13Z" message: Rollout is not healthy reason: RolloutHealthy status: "False" type: Healthy - lastTransitionTime: "2024-10-30T14:43:13Z" lastUpdateTime: "2024-10-30T14:43:13Z" message: RolloutCompleted reason: RolloutCompleted status: "False" type: Completed - lastTransitionTime: "2024-10-30T14:43:12Z" lastUpdateTime: "2024-10-30T14:43:14Z" message: Created new replica set "alb-rollout-6c6554945c" reason: NewReplicaSetCreated status: "True" type: Progressing currentPodHash: 6c6554945c currentStepHash: 558cc8bf4d currentStepIndex: 0 message: more replicas need to be updated observedGeneration: "3" phase: Progressing readyReplicas: 1 replicas: 1 selector: app=alb-rollout stableRS: d6f4fdcdc 2024-10-30T14:43:12Z Normal RolloutAddedToInformer Rollout resource added to informer: argo-rollouts/alb-rollout 2024-10-30T14:43:12Z Normal RolloutNotCompleted Rollout not completed, started update to revision 1 (d6f4fdcdc) 2024-10-30T14:43:12Z Normal RolloutUpdated Rollout updated to revision 1 2024-10-30T14:43:12Z Normal NewReplicaSetCreated Created ReplicaSet alb-rollout-d6f4fdcdc (revision 1) 2024-10-30T14:43:15Z (x2) Normal PatchingALBIngress Updating Ingress `alb-rollout-multi-ingress-1` to desiredWeight '0' 2024-10-30T14:43:15Z (x2) Normal PatchingALBIngress Updating Ingress `alb-rollout-multi-ingress-2` to desiredWeight '0' 2024-10-30T14:43:12Z (x2) Normal TrafficWeightUpdated Traffic weight updated to 0 2024-10-30T14:43:12Z Normal ScalingReplicaSet Scaled up ReplicaSet alb-rollout-d6f4fdcdc (revision 1) from 0 to 1 2024-10-30T14:43:12Z Normal RolloutCompleted Rollout completed update to revision 1 (d6f4fdcdc): Initial deploy 2024-10-30T14:43:14Z (x2) Normal TrafficWeightUpdated Traffic weight updated 2024-10-30T14:43:13Z Normal SwitchService Switched selector for service 'alb-rollout-stable' from '' to 'd6f4fdcdc' 2024-10-30T14:43:13Z Normal SwitchService Switched selector for service 'alb-rollout-canary' from '' to 'd6f4fdcdc' 2024-10-30T14:43:13Z Normal RolloutNotCompleted Rollout not completed, started update to revision 2 (6c6554945c) 2024-10-30T14:43:14Z Normal RolloutUpdated Rollout updated to revision 2 2024-10-30T14:43:14Z Normal NewReplicaSetCreated Created ReplicaSet alb-rollout-6c6554945c (revision 2) 2024-10-30T14:43:14Z Normal ExperimentCreated Created Experiment 'alb-rollout-6c6554945c-2-0' 2024-10-30T14:43:15Z Normal TrafficWeightUpdated Traffic weight updated additional: [{20 } {20 }] time="2024-10-30T14:43:24Z" level=info msg="Deleting e2e-test-name=TestALBExperimentStepNoSetWeightMultiIngress" time="2024-10-30T14:43:24Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress (18.30s) === FAIL: test/e2e TestAWSSuite (re-run 1) (18.69s) === FAIL: test/e2e TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 1) (36.42s) time="2024-10-30T14:43:35Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:43:36Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-10-30T14:43:37Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-10-30T14:43:37Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-10-30T14:44:08Z" level=info msg="Condition 'phase=Successful' met after 30s" time="2024-10-30T14:44:08Z" level=info msg="Sleeping 3s" time="2024-10-30T14:44:11Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-10-30T14:44:11Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-10-30T14:44:11Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 34s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 34s delay:26s │ └──□ experiment-with-dry-run-baseline-9wfmw Pod ✔ Running 34s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ e0b19688-4b81-40e1-a00c-4f4e770d0f7d.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-10-30T14:43:37Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "10278102" uid: b39385ed-c1ee-4091-b7d4-0b77a60531ff spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-10-30T14:43:38Z" conditions: - lastTransitionTime: "2024-10-30T14:44:08Z" lastUpdateTime: "2024-10-30T14:44:08Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-10-30T14:44:08Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-10-30T14:43:37Z (x2) Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-10-30T14:43:37Z (x2) Normal ExperimentPending Experiment transitioned from -> Pending 2024-10-30T14:43:37Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-10-30T14:43:38Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-10-30T14:43:38Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-10-30T14:43:38Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-10-30T14:43:38Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-10-30T14:44:08Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-10-30T14:44:08Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-10-30T14:44:08Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-10-30T14:44:11Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:44:11Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (36.42s) === FAIL: test/e2e TestExperimentSuite (re-run 1) (36.80s) === FAIL: test/e2e TestFunctionalSuite/TestControllerMetrics (re-run 1) (0.53s) time="2024-10-30T14:44:23Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-10-30T14:44:23Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 100 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc000501860, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc000bc7d00) /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc0005a3e40?, 0xc000986228?, 0xc000a0d4f0?}, {0x3212d41, 0x4}, {0xc0007dde70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc0005a3e40?, 0xc000986228?, 0xc000bc7d00?}, {0xc0007dde70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc000501860) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc000501860, 0xc0008d9200) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 15 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-10-30T14:44:23Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.53s) === FAIL: test/e2e TestFunctionalSuite (re-run 1) (1.99s) time="2024-10-30T14:44:23Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" === FAIL: test/e2e TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 2) (4.66s) time="2024-10-30T14:44:33Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:44:35Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-10-30T14:44:36Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-10-30T14:44:37Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:44:38Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:44:38Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-10-30T14:44:38Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:44:38Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (4.66s) === FAIL: test/e2e TestAPISIXSuite (re-run 2) (5.09s) === FAIL: test/e2e TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 2) (37.58s) time="2024-10-30T14:45:21Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:45:23Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-10-30T14:45:24Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-10-30T14:45:24Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-10-30T14:45:56Z" level=info msg="Condition 'phase=Successful' met after 31s" time="2024-10-30T14:45:56Z" level=info msg="Sleeping 3s" time="2024-10-30T14:45:59Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-10-30T14:45:59Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-10-30T14:45:59Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 35s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 35s delay:26s │ └──□ experiment-with-dry-run-baseline-s9h9p Pod ✔ Running 35s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ 61d7421f-1e4e-4f4f-928b-08511edb1a20.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-10-30T14:45:24Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "10279234" uid: 95d9950d-2c96-4ae1-88a6-465495232909 spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-10-30T14:45:26Z" conditions: - lastTransitionTime: "2024-10-30T14:45:56Z" lastUpdateTime: "2024-10-30T14:45:56Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-10-30T14:45:56Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-10-30T14:45:24Z (x2) Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-10-30T14:45:24Z (x2) Normal ExperimentPending Experiment transitioned from -> Pending 2024-10-30T14:45:24Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-10-30T14:45:26Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-10-30T14:45:26Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-10-30T14:45:26Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-10-30T14:45:26Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-10-30T14:45:56Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-10-30T14:45:56Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-10-30T14:45:56Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-10-30T14:45:59Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:45:59Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (37.58s) === FAIL: test/e2e TestExperimentSuite (re-run 2) (37.97s) === FAIL: test/e2e TestFunctionalSuite/TestControllerMetrics (re-run 2) (0.81s) time="2024-10-30T14:46:11Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-10-30T14:46:11Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 114 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc000aae340, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc0006b3f00) /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc0007914c0?, 0xc0001ab5f8?, 0xc00087d650?}, {0x3212d41, 0x4}, {0xc000b83e70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc0007914c0?, 0xc0001ab5f8?, 0xc0006b3f00?}, {0xc000b83e70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc000aae340) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc000aae340, 0xc00087eab0) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 83 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-10-30T14:46:11Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.81s) === FAIL: test/e2e TestFunctionalSuite (re-run 2) (2.28s) time="2024-10-30T14:46:11Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" === FAIL: test/e2e TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 3) (4.93s) time="2024-10-30T14:46:22Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:46:23Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-10-30T14:46:24Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-10-30T14:46:25Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:46:26Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:46:26Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-10-30T14:46:26Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:46:27Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (4.93s) === FAIL: test/e2e TestAPISIXSuite (re-run 3) (5.30s) === FAIL: test/e2e TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 3) (37.23s) time="2024-10-30T14:46:37Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:46:38Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-10-30T14:46:39Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-10-30T14:46:39Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-10-30T14:47:11Z" level=info msg="Condition 'phase=Successful' met after 31s" time="2024-10-30T14:47:11Z" level=info msg="Sleeping 3s" time="2024-10-30T14:47:14Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-10-30T14:47:14Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-10-30T14:47:14Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 35s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 35s delay:26s │ └──□ experiment-with-dry-run-baseline-g6ll6 Pod ✔ Running 35s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ 69a022cc-dc98-48cc-8686-d40fdd4763f7.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-10-30T14:46:39Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "10279817" uid: d506ad7e-6feb-46bb-b266-97f09d2e3d1a spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-10-30T14:46:41Z" conditions: - lastTransitionTime: "2024-10-30T14:47:11Z" lastUpdateTime: "2024-10-30T14:47:11Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-10-30T14:47:11Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-10-30T14:46:39Z (x2) Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-10-30T14:46:39Z (x2) Normal ExperimentPending Experiment transitioned from -> Pending 2024-10-30T14:46:39Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-10-30T14:46:41Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-10-30T14:46:41Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-10-30T14:46:41Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-10-30T14:46:41Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-10-30T14:47:11Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-10-30T14:47:11Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-10-30T14:47:11Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-10-30T14:47:14Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:47:14Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (37.23s) === FAIL: test/e2e TestExperimentSuite (re-run 3) (37.59s) === FAIL: test/e2e TestFunctionalSuite/TestControllerMetrics (re-run 3) (0.65s) time="2024-10-30T14:47:25Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-10-30T14:47:26Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 54 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc0007829c0, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc000c04000) /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc000c0d100?, 0xc000c09728?, 0xc000cbda80?}, {0x3212d41, 0x4}, {0xc0007bbe70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc000c0d100?, 0xc000c09728?, 0xc000c04000?}, {0xc0007bbe70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc0007829c0) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc0007829c0, 0xc0004e4bd0) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 37 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-10-30T14:47:26Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.65s) === FAIL: test/e2e TestFunctionalSuite (re-run 3) (2.27s) time="2024-10-30T14:47:25Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" === FAIL: test/e2e TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 4) (4.99s) time="2024-10-30T14:47:36Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:47:38Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-10-30T14:47:39Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-10-30T14:47:40Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:47:41Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:47:41Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-10-30T14:47:41Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:47:41Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (4.99s) === FAIL: test/e2e TestAPISIXSuite (re-run 4) (5.39s) === FAIL: test/e2e TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 4) (36.31s) time="2024-10-30T14:47:52Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:47:53Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-10-30T14:47:54Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-10-30T14:47:54Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-10-30T14:48:25Z" level=info msg="Condition 'phase=Successful' met after 30s" time="2024-10-30T14:48:25Z" level=info msg="Sleeping 3s" time="2024-10-30T14:48:28Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-10-30T14:48:28Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-10-30T14:48:28Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 34s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 34s delay:26s │ └──□ experiment-with-dry-run-baseline-xrffd Pod ✔ Running 34s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ b68d9d01-d22a-4ef8-b1ff-3ae24ca5763d.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-10-30T14:47:54Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "10280805" uid: 75b788be-0f28-4a04-a13a-747befb476f9 spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-10-30T14:47:55Z" conditions: - lastTransitionTime: "2024-10-30T14:48:25Z" lastUpdateTime: "2024-10-30T14:48:25Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-10-30T14:48:25Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-10-30T14:47:54Z (x2) Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-10-30T14:47:54Z (x2) Normal ExperimentPending Experiment transitioned from -> Pending 2024-10-30T14:47:54Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-10-30T14:47:55Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-10-30T14:47:55Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-10-30T14:47:55Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-10-30T14:47:55Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-10-30T14:48:25Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-10-30T14:48:25Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-10-30T14:48:25Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-10-30T14:48:28Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:48:28Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (36.31s) === FAIL: test/e2e TestExperimentSuite (re-run 4) (36.68s) === FAIL: test/e2e TestFunctionalSuite/TestControllerMetrics (re-run 4) (0.61s) time="2024-10-30T14:48:40Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-10-30T14:48:40Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 89 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc000102ea0, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc000584f00) /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc000b07b00?, 0xc000a34168?, 0xc000b402d0?}, {0x3212d41, 0x4}, {0xc00087be70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc000b07b00?, 0xc000a34168?, 0xc000584f00?}, {0xc00087be70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc000102ea0) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc000102ea0, 0xc000ad7440) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 15 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-10-30T14:48:40Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.61s) === FAIL: test/e2e TestFunctionalSuite (re-run 4) (2.09s) time="2024-10-30T14:48:40Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" === FAIL: test/e2e TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (re-run 5) (4.70s) time="2024-10-30T14:48:50Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:48:52Z" level=info msg="service/rollout-apisix-canary-canary created\n" rollout=rollout-apisix-canary time="2024-10-30T14:48:53Z" level=info msg="service/rollout-apisix-canary-stable created\n" rollout=rollout-apisix-canary time="2024-10-30T14:48:54Z" level=info msg="apisixroute.apisix.apache.org/rollouts-apisix-route unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:48:55Z" level=info msg="rollout.argoproj.io/rollout-apisix-canary unchanged\n" rollout=rollout-apisix-canary time="2024-10-30T14:48:55Z" level=info msg="Waiting for condition: status=Healthy" rollout=rollout-apisix-canary when.go:423: rollouts.argoproj.io "rollout-apisix-canary" not found time="2024-10-30T14:48:55Z" level=info msg="Deleting e2e-test-name=TestAPISIXCanarySetHeaderStep" time="2024-10-30T14:48:55Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (4.70s) === FAIL: test/e2e TestAPISIXSuite (re-run 5) (5.03s) === FAIL: test/e2e TestExperimentSuite/TestExperimentWithDryRunMetrics (re-run 5) (36.77s) time="2024-10-30T14:49:05Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:49:07Z" level=info msg="analysistemplate.argoproj.io/dry-run-job created\n" time="2024-10-30T14:49:08Z" level=info msg="experiment.argoproj.io/experiment-with-dry-run created\n" time="2024-10-30T14:49:08Z" level=info msg="Waiting for Experiment experiment-with-dry-run condition: phase=Successful" time="2024-10-30T14:49:39Z" level=info msg="Condition 'phase=Successful' met after 30s" time="2024-10-30T14:49:39Z" level=info msg="Sleeping 3s" time="2024-10-30T14:49:42Z" level=info msg="Expected Dry-Run Summary: Count=1, Successful=0, Failed=1, Error=0, Inconclusive=0" time="2024-10-30T14:49:42Z" level=info msg="Analysis Dry-Run Summary: Count=1, Successful=1, Failed=0, Error=0, Inconclusive=0" time="2024-10-30T14:49:42Z" level=error msg="Dry-Run Summary of AnalysisRun: 'experiment-with-dry-run-dry-run-job' doesn't match the expectations" Name: experiment-with-dry-run Namespace: argo-rollouts Status: ✔ Successful Images: nginx:1.19-alpine (Σ:baseline) NAME KIND STATUS AGE INFO Σ experiment-with-dry-run Experiment ✔ Successful 34s ├──⧉ experiment-with-dry-run-baseline ReplicaSet ✔ Healthy 34s delay:26s │ └──□ experiment-with-dry-run-baseline-f6nkh Pod ✔ Running 34s ready:1/1 └──α experiment-with-dry-run-dry-run-job AnalysisRun ✔ Successful 33s ✔ 1 └──⊞ 8b04005c-0639-4900-81dd-22ea7dc1e63a.test-1.1 Job ✔ Successful 33s --- apiVersion: argoproj.io/v1alpha1 kind: Experiment metadata: creationTimestamp: "2024-10-30T14:49:08Z" generation: 9 labels: argo-rollouts.argoproj.io/controller-instance-id: argo-rollouts-e2e e2e-test-name: TestExperimentWithDryRunMetrics name: experiment-with-dry-run namespace: argo-rollouts resourceVersion: "10281497" uid: 3c4d8d24-8c1a-4a78-99bf-f046ca53b8ac spec: analyses: - name: dry-run-job templateName: dry-run-job dryRun: - metricName: test.* duration: 30s progressDeadlineSeconds: 30 templates: - name: baseline replicas: 1 selector: matchLabels: app: experiment-with-dry-run service: {} template: metadata: creationTimestamp: null labels: app: experiment-with-dry-run spec: containers: - image: nginx:1.19-alpine name: experiment-with-dry-run resources: requests: cpu: 1m memory: 16Mi status: analysisRuns: - analysisRun: experiment-with-dry-run-dry-run-job name: dry-run-job phase: Running availableAt: "2024-10-30T14:49:09Z" conditions: - lastTransitionTime: "2024-10-30T14:49:39Z" lastUpdateTime: "2024-10-30T14:49:39Z" message: Experiment "experiment-with-dry-run" has successfully ran and completed. reason: ExperimentCompleted status: "False" type: Progressing phase: Successful templateStatuses: - availableReplicas: 1 lastTransitionTime: "2024-10-30T14:49:39Z" name: baseline readyReplicas: 1 replicas: 1 status: Successful updatedReplicas: 1 2024-10-30T14:49:08Z (x2) Normal TemplateProgressing Template 'baseline' transitioned from -> Progressing 2024-10-30T14:49:08Z (x2) Normal ExperimentPending Experiment transitioned from -> Pending 2024-10-30T14:49:08Z Normal ScalingReplicaSet Scaled up ReplicaSet experiment-with-dry-run-baseline from 0 to 1 2024-10-30T14:49:09Z Normal TemplateRunning Template 'baseline' transitioned from Progressing -> Running 2024-10-30T14:49:09Z Normal ExperimentRunning Experiment transitioned from Pending -> Running 2024-10-30T14:49:09Z Normal AnalysisRunPending AnalysisRun 'dry-run-job' transitioned from -> Pending 2024-10-30T14:49:09Z Normal AnalysisRunRunning AnalysisRun 'dry-run-job' transitioned from -> Running 2024-10-30T14:49:39Z Normal TemplateSuccessful Template 'baseline' transitioned from Running -> Successful 2024-10-30T14:49:39Z Normal ExperimentSuccessful Experiment transitioned from Running -> Successful 2024-10-30T14:49:39Z Normal AnalysisRunTerminating Terminating dry-run-job (experiment-with-dry-run-dry-run-job) time="2024-10-30T14:49:42Z" level=info msg="Deleting e2e-test-name=TestExperimentWithDryRunMetrics" time="2024-10-30T14:49:42Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (36.77s) === FAIL: test/e2e TestExperimentSuite (re-run 5) (37.15s) === FAIL: test/e2e TestFunctionalSuite/TestControllerMetrics (re-run 5) (0.71s) time="2024-10-30T14:49:54Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" functional_test.go:1402: Error Trace: /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1402 Error: Received unexpected error: Get "http://localhost:8090/metrics": dial tcp [::1]:8090: connect: connection refused Test: TestFunctionalSuite/TestControllerMetrics time="2024-10-30T14:49:54Z" level=info msg="Deleting e2e-test-name=TestControllerMetrics" suite.go:87: test panicked: runtime error: invalid memory address or nil pointer dereference goroutine 30 [running]: runtime/debug.Stack() /usr/local/go/src/runtime/debug/stack.go:24 +0x5e github.com/stretchr/testify/suite.failOnPanic(0xc000502d00, {0x2ea07e0, 0x4cf8420}) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:87 +0x39 github.com/stretchr/testify/suite.Run.func1.1() /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:183 +0x24c panic({0x2ea07e0?, 0x4cf8420?}) /usr/local/go/src/runtime/panic.go:914 +0x21f github.com/argoproj/argo-rollouts/test/e2e.(*FunctionalSuite).TestControllerMetrics(0xc0001cdc00) /tmp/tmp.SIcIuinRDW/argo-rollouts/test/e2e/functional_test.go:1403 +0xc4 reflect.Value.call({0xc000a078c0?, 0xc000992170?, 0xc000741de0?}, {0x3212d41, 0x4}, {0xc000a91e70, 0x1, 0x2b4e148?}) /usr/local/go/src/reflect/value.go:596 +0xce7 reflect.Value.Call({0xc000a078c0?, 0xc000992170?, 0xc0001cdc00?}, {0xc000a91e70?, 0x1207aed?, 0x4460798?}) /usr/local/go/src/reflect/value.go:380 +0xb9 github.com/stretchr/testify/suite.Run.func1(0xc000502d00) /root/go/pkg/mod/github.com/stretchr/testify@v1.8.4/suite/suite.go:197 +0x467 testing.tRunner(0xc000502d00, 0xc00003f5f0) /usr/local/go/src/testing/testing.go:1595 +0xff created by testing.(*T).Run in goroutine 83 /usr/local/go/src/testing/testing.go:1648 +0x3ad time="2024-10-30T14:49:54Z" level=info msg="Deleting e2e-test-name" --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.71s) === FAIL: test/e2e TestFunctionalSuite (re-run 5) (2.21s) time="2024-10-30T14:49:54Z" level=info msg="analysistemplate.argoproj.io/sleep-job created\n" DONE 6 runs, 138 tests, 6 skipped, 40 failures in 2855.794s make: *** [Makefile:239: test-e2e] Error 1 + set +x ----------------------------------------------------------------- These were the tests that succeeded: PASS test/e2e PASS test/e2e.TestAPISIXSuite/TestAPISIXCanaryStep (28.36s) PASS test/e2e.TestAWSSuite (re-run 2) (23.15s) PASS test/e2e.TestAWSSuite/TestALBExperimentStep (24.92s) PASS test/e2e.TestAWSSuite/TestALBExperimentStepMultiIngress (25.13s) PASS test/e2e.TestAWSSuite/TestALBExperimentStepNoSetWeight (23.96s) PASS test/e2e.TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress (re-run 2) (22.76s) PASS test/e2e.TestAWSSuite/TestALBPingPongUpdate (15.72s) PASS test/e2e.TestAWSSuite/TestALBPingPongUpdateMultiIngress (15.09s) PASS test/e2e.TestAWSSuite/TestAlbHeaderRoute (13.28s) PASS test/e2e.TestAWSSuite/TestAlbHeaderRouteMultiIngress (25.18s) PASS test/e2e.TestAnalysisSuite (323.17s) PASS test/e2e.TestAnalysisSuite/TestAnalysisWithArgs (42.04s) PASS test/e2e.TestAnalysisSuite/TestAnalysisWithSecret (31.97s) PASS test/e2e.TestAnalysisSuite/TestBackgroundAnalysisWithArgs (31.13s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenAbortAndUpdate (22.25s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenAnalysis (30.96s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenKitchenSink (47.44s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenPostPromotionFail (15.72s) PASS test/e2e.TestAnalysisSuite/TestBlueGreenPrePromotionFail (28.33s) PASS test/e2e.TestAnalysisSuite/TestCanaryBackgroundAnalysis (4.74s) PASS test/e2e.TestAnalysisSuite/TestCanaryFailInlineMultipleAnalysis (13.05s) PASS test/e2e.TestAnalysisSuite/TestCanaryInlineAnalysis (15.06s) PASS test/e2e.TestAnalysisSuite/TestCanaryInlineMultipleAnalysis (17.07s) PASS test/e2e.TestAnalysisSuite/TestMultipleAnalysis (9.89s) PASS test/e2e.TestBlueGreenSuite (145.28s) PASS test/e2e.TestBlueGreenSuite/TestBlueGreenPreviewReplicaCount (7.26s) PASS test/e2e.TestBlueGreenSuite/TestBlueGreenProgressDeadlineExceededWithPause (63.88s) PASS test/e2e.TestBlueGreenSuite/TestBlueGreenProgressDeadlineExceededWithoutPause (63.88s) PASS test/e2e.TestBlueGreenSuite/TestEphemeralMetadata (9.84s) PASS test/e2e.TestCanarySuite (366.27s) PASS test/e2e.TestCanarySuite/TestCanaryDynamicStableScale (23.54s) PASS test/e2e.TestCanarySuite/TestCanaryDynamicStableScaleRollbackToStable (17.24s) PASS test/e2e.TestCanarySuite/TestCanaryProgressDeadlineExceededWithPause (123.69s) PASS test/e2e.TestCanarySuite/TestCanaryScaleDownDelay (14.38s) PASS test/e2e.TestCanarySuite/TestCanaryScaleDownOnAbort (12.00s) PASS test/e2e.TestCanarySuite/TestCanaryScaleDownOnAbortNoTrafficRouting (11.11s) PASS test/e2e.TestCanarySuite/TestCanarySetCanaryScale (12.70s) PASS test/e2e.TestCanarySuite/TestCanaryUnScaleDownOnAbort (11.67s) PASS test/e2e.TestCanarySuite/TestCanaryWithPausedRollout (102.16s) PASS test/e2e.TestCanarySuite/TestEphemeralMetadata (7.10s) PASS test/e2e.TestCanarySuite/TestReduceWeightAndHonorMaxUnavailable (27.07s) PASS test/e2e.TestCanarySuite/TestRolloutScalingDuringUpdate (4.45s) PASS test/e2e.TestCanarySuite/TestRolloutScalingWhenPaused (6.43s) PASS test/e2e.TestCanarySuite/TestRolloutWithMaxSurgeScalingDuringUpdate (3.53s) PASS test/e2e.TestExperimentSuite/TestExperimentWithMeasurementRetentionMetrics (36.77s) PASS test/e2e.TestExperimentSuite/TestExperimentWithMultiportServiceAndScaleDownDelay (18.08s) PASS test/e2e.TestExperimentSuite/TestExperimentWithServiceAndScaleDownDelay (17.97s) PASS test/e2e.TestExperimentSuite/TestExperimentWithServiceNameAndScaleDownDelay (17.07s) PASS test/e2e.TestExperimentSuite/TestRolloutWithExperimentAndAnalysis (10.27s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenExceedProgressDeadlineAbort (10.26s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenPreviewReplicaCountPromoteFull (8.63s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenPromoteFull (5.79s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenScaleDownDelay (9.59s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenScaleDownOnAbort (8.90s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenToCanary (5.05s) PASS test/e2e.TestFunctionalSuite/TestBlueGreenUpdate (5.92s) PASS test/e2e.TestFunctionalSuite/TestCanaryPromoteFull (9.92s) PASS test/e2e.TestFunctionalSuite/TestCompleteRolloutRestart (22.00s) PASS test/e2e.TestFunctionalSuite/TestContainerResourceFormats (3.07s) PASS test/e2e.TestFunctionalSuite/TestFixInvalidSpec (21.84s) PASS test/e2e.TestFunctionalSuite/TestKubectlWaitForCompleted (23.92s) PASS test/e2e.TestFunctionalSuite/TestKubectlWaitForPaused (7.31s) PASS test/e2e.TestFunctionalSuite/TestMalformedRollout (3.46s) PASS test/e2e.TestFunctionalSuite/TestMalformedRolloutEphemeralCtr (3.01s) PASS test/e2e.TestFunctionalSuite/TestRolloutAbortRetryPromote (9.92s) PASS test/e2e.TestFunctionalSuite/TestRolloutBadToBadUpdate (22.95s) PASS test/e2e.TestFunctionalSuite/TestRolloutBadToGoodSetWeightUpdate (23.41s) PASS test/e2e.TestFunctionalSuite/TestRolloutBadToGoodUpdate (18.06s) PASS test/e2e.TestFunctionalSuite/TestRolloutGoodToBadUpdate (25.54s) PASS test/e2e.TestFunctionalSuite/TestRolloutPDBRestart (46.78s) PASS test/e2e.TestFunctionalSuite/TestRolloutPauseDurationGreaterThanProgressDeadlineSeconds (37.51s) PASS test/e2e.TestFunctionalSuite/TestRolloutPodVolumesItemsSpec (9.49s) PASS test/e2e.TestFunctionalSuite/TestRolloutRestart (18.05s) PASS test/e2e.TestFunctionalSuite/TestWorkloadRef (19.24s) PASS test/e2e.TestFunctionalSuite/TestWorkloadRefTemplate (2.89s) PASS test/e2e.TestHeaderRoutingSuite (15.60s) PASS test/e2e.TestHeaderRoutingSuite/TestIstioHostHeaderRoute (15.26s) PASS test/e2e.TestIstioSuite (111.29s) PASS test/e2e.TestIstioSuite/TestIstioAbortUpdate (7.48s) PASS test/e2e.TestIstioSuite/TestIstioAbortUpdateDeleteAllCanaryPods (9.52s) PASS test/e2e.TestIstioSuite/TestIstioHostSplit (18.15s) PASS test/e2e.TestIstioSuite/TestIstioHostSplitExperimentStep (19.10s) PASS test/e2e.TestIstioSuite/TestIstioHostSplitOnlyTls (8.91s) PASS test/e2e.TestIstioSuite/TestIstioSubsetSplit (10.60s) PASS test/e2e.TestIstioSuite/TestIstioSubsetSplitExperimentStep (19.22s) PASS test/e2e.TestIstioSuite/TestIstioSubsetSplitSingleRoute (8.83s) PASS test/e2e.TestIstioSuite/TestIstioUpdateInMiddleZeroCanaryReplicas (9.05s) PASS test/e2e.TestMirrorRouteSuite (16.00s) PASS test/e2e.TestMirrorRouteSuite/TestIstioHostMirrorRoute (15.63s) PASS test/e2e.TestRollbackSuite (57.61s) PASS test/e2e.TestRollbackSuite/TestRollbackAnalysisOutsideWindow (30.83s) PASS test/e2e.TestRollbackSuite/TestRollbackAnalysisWithinWindow (24.10s) These were the tests that failed: --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (4.66s) --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (4.70s) --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (4.93s) --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (4.99s) --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (5.03s) --- FAIL: TestAPISIXSuite/TestAPISIXCanarySetHeaderStep (7.60s) --- FAIL: TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress (18.30s) --- FAIL: TestAWSSuite/TestALBExperimentStepNoSetWeightMultiIngress (18.48s) --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (36.31s) --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (36.42s) --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (36.77s) --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (36.79s) --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (37.23s) --- FAIL: TestExperimentSuite/TestExperimentWithDryRunMetrics (37.58s) --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.53s) --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.61s) --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.64s) --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.65s) --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.71s) --- FAIL: TestFunctionalSuite/TestControllerMetrics (0.81s) E2E tests file is complete: DONE 6 runs, 138 tests, 6 skipped, 40 failures in 2855.794s * SUCCESS: No unexpected errors occurred. * Cleaning up hack/run-upstream-argo-rollouts-e2e-tests.sh: line 7: killall: command not found hack/run-upstream-argo-rollouts-e2e-tests.sh: line 8: killall: command not found + [[ 0 != 0 ]] + kubectl delete ns argo-rollouts namespace "argo-rollouts" deleted + kubectl wait --timeout=5m --for=delete namespace/argo-rollouts + kubectl create ns argo-rollouts namespace/argo-rollouts created + kubectl config set-context --current --namespace=argo-rollouts Context "default/api-psi-416-ocp-gitops-qe-com:6443/kube:admin" modified. + cat + kubectl apply -f /tmp/tmp.mWLvz2au9x/rollout-manager.yaml rolloutmanager.argoproj.io/argo-rollout created + cd /tmp/tmp.mWLvz2au9x + git clone https://github.com/argoproj-labs/rollouts-plugin-trafficrouter-openshift Cloning into 'rollouts-plugin-trafficrouter-openshift'... + cd /tmp/tmp.mWLvz2au9x/rollouts-plugin-trafficrouter-openshift + git checkout 8b4125a7f9ecffb0247df91a4c890f88c0c523b7 Note: switching to '8b4125a7f9ecffb0247df91a4c890f88c0c523b7'. You are in 'detached HEAD' state. You can look around, make experimental changes and commit them, and you can discard any commits you make in this state without impacting any branches by switching back to a branch. If you want to create a new branch to retain commits you create, you may do so (now or later) by using -c with the switch command. Example: git switch -c Or undo this operation with: git switch - Turn off this advice by setting config variable advice.detachedHead to false HEAD is now at 8b4125a Switch lint to use Makefile, fix go.mod + make test-e2e go test -v -p=1 -timeout=20m -race -count=1 -coverprofile=coverage.out ./tests/e2e go: downloading go1.22.2 (linux/amd64) go: downloading k8s.io/client-go v0.26.3 go: downloading github.com/openshift/api v0.0.0-20230417092139-1b2161d23365 go: downloading github.com/argoproj/argo-rollouts v1.6.6 go: downloading k8s.io/apimachinery v0.26.3 go: downloading github.com/onsi/gomega v1.33.1 go: downloading github.com/onsi/ginkgo/v2 v2.19.0 go: downloading github.com/openshift/client-go v0.0.0-20230419131419-497c7032c581 go: downloading k8s.io/api v0.26.3 go: downloading github.com/google/go-cmp v0.6.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading golang.org/x/net v0.25.0 go: downloading golang.org/x/sys v0.20.0 go: downloading k8s.io/klog/v2 v2.110.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading golang.org/x/text v0.15.0 go: downloading k8s.io/cli-runtime v0.26.3 go: downloading k8s.io/kubernetes v1.26.3 go: downloading google.golang.org/grpc v1.57.1 go: downloading golang.org/x/term v0.20.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/xlab/treeprint v1.2.0 go: downloading k8s.io/apiserver v0.26.3 go: downloading k8s.io/component-base v0.26.3 go: downloading k8s.io/component-helpers v0.26.3 === RUN TestPlugin Running Suite: Plugin e2e Tests Suite - /tmp/tmp.mWLvz2au9x/rollouts-plugin-trafficrouter-openshift/tests/e2e ============================================================================================================= Random Seed: 1730300201 Will run 3 of 3 specs service/rollouts-demo-stable created service/rollouts-demo-canary created route.route.openshift.io/rollouts-demo created rollout.argoproj.io/rollouts-demo created Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy W1030 14:57:02.606642 70498 warnings.go:70] unknown field "spec.template.metadata.creationTimestamp" Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Canary weight mismatch: got 20, want 70 Canary weight mismatch: got 20, want 70 Canary weight mismatch: got 20, want 70 Canary weight mismatch: got 20, want 70 Canary weight mismatch: got 20, want 70 Canary weight mismatch: got 20, want 70 Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy •service/rollouts-demo-stable created service/rollouts-demo-canary created route.route.openshift.io/rollouts-demo-prod created route.route.openshift.io/rollouts-demo-stage created rollout.argoproj.io/rollouts-demo created Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy W1030 14:57:30.190852 70498 warnings.go:70] unknown field "spec.template.metadata.creationTimestamp" Rollout phase mismatch, got Healthy, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Canary weight mismatch: got 100, want 0 •service/rollouts-demo-stable created service/rollouts-demo-canary created route.route.openshift.io/rollouts-demo created analysistemplate.argoproj.io/sample-template created rollout.argoproj.io/rollouts-demo created Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy W1030 14:57:55.800185 70498 warnings.go:70] unknown field "spec.template.metadata.creationTimestamp" Rollout phase mismatch, got Healthy, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Paused Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy Rollout phase mismatch, got Progressing, want Healthy • Ran 3 of 3 Specs in 88.776 seconds SUCCESS! -- 3 Passed | 0 Failed | 0 Pending | 0 Skipped --- PASS: TestPlugin (88.78s) PASS coverage: [no statements] ok github.com/argoproj-labs/rollouts-plugin-trafficrouter-openshift/tests/e2e 89.979s coverage: [no statements] + [[ 0 != 0 ]] + oc patch subscription openshift-gitops-operator -n openshift-operators --type json --patch '[{"op": "remove", "path": "/spec/config"}]' subscription.operators.coreos.com/openshift-gitops-operator patched + update_result + [[ 2 != 0 ]] + echo test-argo-rollouts + tee /install-dir/psi-416-result.txt test-argo-rollouts + exit 0