[jianl@jianl-thinkpadt14gen4 418]$ oc get clusterversion NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.18.0-rc.4 True False 28m Cluster version is 4.18.0-rc.4 [jianl@jianl-thinkpadt14gen4 418]$ oc create namespace fault-injection namespace/fault-injection created [jianl@jianl-thinkpadt14gen4 418]$ [jianl@jianl-thinkpadt14gen4 418]$ oc apply -n fault-injection -f ~/tests/1379/01-fault-injecting-proxy.yaml deployment.apps/fault-injecting-proxy created service/fault-injecting-proxy created configmap/nginx-config created [jianl@jianl-thinkpadt14gen4 418]$ [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection get pod NAME READY STATUS RESTARTS AGE fault-injecting-proxy-987695595-5m89j 2/2 Running 0 36s [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection logs pod/fault-injecting-proxy-987695595-5m89j -c nginx /docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration /docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/ /docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh 10-listen-on-ipv6-by-default.sh: info: Getting the checksum of /etc/nginx/conf.d/default.conf 10-listen-on-ipv6-by-default.sh: info: /etc/nginx/conf.d/default.conf differs from the packaged version /docker-entrypoint.sh: Sourcing /docker-entrypoint.d/15-local-resolvers.envsh /docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh /docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh /docker-entrypoint.sh: Configuration complete; ready for start up [jianl@jianl-thinkpadt14gen4 418]$ [jianl@jianl-thinkpadt14gen4 418]$ [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection logs pod/fault-injecting-proxy-987695595-5m89j -c toxiproxy {"level":"info","version":"2.11.0","caller":"server.go:78","time":"2025-01-15T02:06:12Z","message":"Starting Toxiproxy"} {"level":"info","address":"0.0.0.0:8474","caller":"api.go:57","time":"2025-01-15T02:06:12Z","message":"Starting Toxiproxy HTTP server"} [jianl@jianl-thinkpadt14gen4 418]$ install OSUS operator by web console [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection get csv NAME DISPLAY VERSION REPLACES PHASE update-service-operator.v5.0.3 OpenShift Update Service 5.0.3 update-service-operator.v5.0.2 Succeeded [jianl@jianl-thinkpadt14gen4 418]$ get service: [jianl@jianl-thinkpadt14gen4 418]$ oc get service -n fault-injection NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE fault-injecting-proxy ClusterIP 172.30.182.194 443/TCP 144m updateservice-operator-metrics ClusterIP 172.30.217.117 8443/TCP 88s [jianl@jianl-thinkpadt14gen4 418]$ [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection get pod NAME READY STATUS RESTARTS AGE fault-injecting-proxy-987695595-5m89j 2/2 Running 0 145m updateservice-operator-697d4bd7f6-w82vj 1/1 Running 0 2m2s [jianl@jianl-thinkpadt14gen4 418]$ install OSUS instance by web console Combine the releases path: fault-injecting-proxy.fault-injection.svc:443/openshift-release-dev/ocp-release [jianl@jianl-thinkpadt14gen4 418]$ oc get updateservice/sample -oyaml apiVersion: updateservice.operator.openshift.io/v1 kind: UpdateService metadata: creationTimestamp: "2025-01-15T04:31:53Z" generation: 1 name: sample namespace: fault-injection resourceVersion: "108257" uid: da30a539-f7fa-420a-8ce8-9481de760856 spec: graphDataImage: quay.io/openshifttest/graph-data:latest releases: fault-injecting-proxy.fault-injection.svc:443/openshift-release-dev/ocp-release replicas: 1 status: conditions: - lastHeartbeatTime: "2025-01-15T04:31:55Z" lastTransitionTime: "2025-01-15T04:31:55Z" message: image.config.openshift.io.Spec.AdditionalTrustedCA.Name not set for image name cluster reason: NotConfigured status: "False" type: RegistryCACertFound - lastHeartbeatTime: "2025-01-15T04:31:55Z" lastTransitionTime: "2025-01-15T04:31:55Z" reason: Success status: "True" type: ReconcileCompleted policyEngineURI: https://sample-route-fault-injection.apps.jianl011501.qe.devcluster.openshift.com [jianl@jianl-thinkpadt14gen4 418]$ [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection get pod NAME READY STATUS RESTARTS AGE fault-injecting-proxy-987695595-5m89j 2/2 Running 0 146m graph-data-tag-digest 1/1 Running 0 59s sample-5c95d7784d-5k8qk 0/2 Running 0 59s sample-8674c5d6-zzhd2 0/2 Running 0 57s updateservice-operator-697d4bd7f6-w82vj 1/1 Running 0 3m52s [jianl@jianl-thinkpadt14gen4 418]$ [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection get deployment NAME READY UP-TO-DATE AVAILABLE AGE fault-injecting-proxy 1/1 1 1 147m sample 0/1 1 0 99s updateservice-operator 1/1 1 1 4m32s [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection logs deployment/sample > ~/tests/1379/fault_injection_log_1.txt [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection scale deployment/updateservice-operator --replicas=0 deployment.apps/updateservice-operator scaled [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection get deployment NAME READY UP-TO-DATE AVAILABLE AGE fault-injecting-proxy 1/1 1 1 150m sample 0/1 1 0 4m51s updateservice-operator 0/0 0 0 7m44s [jianl@jianl-thinkpadt14gen4 418]$ [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection get configmap NAME DATA AGE kube-root-ca.crt 1 154m nginx-config 1 151m openshift-service-ca.crt 1 154m sample-config 1 5m41s sample-env 7 5m41s [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection edit configmap/sample-config [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection scale deployment/sample --replicas=0 deployment.apps/sample scaled [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection scale deployment/sample --replicas=1 deployment.apps/sample scaled [jianl@jianl-thinkpadt14gen4 418]$ [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection get pods NAME READY STATUS RESTARTS AGE fault-injecting-proxy-987695595-5m89j 2/2 Running 0 155m graph-data-tag-digest 0/1 Completed 0 9m54s sample-8674c5d6-r68ff 0/2 Running 0 2m16s [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection logs pods/sample-8674c5d6-r68ff [2025-01-15T04:39:37Z WARN cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] error fetching manifest and manifestref for openshift-release-dev/ocp-release:4.10.20-s390x: fetching manifest and manifestref for openshift-release-dev/ocp-release:4.10.20-s390x: unexpected HTTP status 502 Bad Gateway, ignoring this image [2025-01-15T04:39:37Z WARN cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] error fetching manifest and manifestref for openshift-release-dev/ocp-release:4.10.21-s390x: fetching manifest and manifestref for openshift-release-dev/ocp-release:4.10.21-s390x: unexpected HTTP status 502 Bad Gateway, ignoring this image [2025-01-15T04:39:37Z WARN cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] error fetching manifest and manifestref for openshift-release-dev/ocp-release:4.10.21-x86_64: fetching manifest and manifestref for openshift-release-dev/ocp-release:4.10.21-x86_64: unexpected HTTP status 502 Bad Gateway, ignoring this image [2025-01-15T04:39:37Z WARN cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] error fetching manifest and manifestref for openshift-release-dev/ocp-release:4.10.22-aarch64: fetching manifest and manifestref for openshift-release-dev/ocp-release:4.10.22-aarch64: unexpected HTTP status 502 Bad Gateway, ignoring this image [2025-01-15T04:39:37Z WARN cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] error fetching manifest and manifestref for openshift-release-dev/ocp-release:4.10.22-ppc64le: fetching manifest and manifestref for openshift-release-dev/ocp-release:4.10.22-ppc64le: unexpected HTTP status 502 Bad Gateway, ignoring this image [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection get deployment NAME READY UP-TO-DATE AVAILABLE AGE fault-injecting-proxy 1/1 1 1 157m sample 1/1 1 1 11m updateservice-operator 0/0 0 0 14m [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection exec deployment/fault-injecting-proxy -c toxiproxy -- /toxiproxy-cli create --listen 0.0.0.0:8080 --upstream quay.io:443 proxied-quay [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection exec deployment/fault-injecting-proxy -c toxiproxy -- /toxiproxy-cli create --listen 0.0.0.0:8080 --upstream quay.io:443 proxied-quay Created new proxy proxied-quay [jianl@jianl-thinkpadt14gen4 418]$ [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection scale deployment/sample --replicas=0 deployment.apps/sample scaled [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection scale deployment/sample --replicas=1 deployment.apps/sample scaled [jianl@jianl-thinkpadt14gen4 418]$ [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection get deployment NAME READY UP-TO-DATE AVAILABLE AGE fault-injecting-proxy 1/1 1 1 163m sample 0/1 1 0 17m updateservice-operator 0/0 0 0 20m [jianl@jianl-thinkpadt14gen4 418]$ [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection get pods NAME READY STATUS RESTARTS AGE fault-injecting-proxy-987695595-5m89j 2/2 Running 0 164m graph-data-tag-digest 0/1 Completed 0 18m sample-8674c5d6-j2jcj 0/2 Running 0 2m27s [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection logs pods/sample-8674c5d6-j2jcj ***********there are no 502 errors anymore ********************* oc -n fault-injection exec deployment/fault-injecting-proxy -c toxiproxy -- /toxiproxy-cli toxic add \ --type reset_peer \ --toxicity 0.1 \ --attribute timeout=50 \ --upstream proxied-quay [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection exec deployment/fault-injecting-proxy -c toxiproxy -- /toxiproxy-cli toxic add \ --type reset_peer \ --toxicity 0.1 \ --attribute timeout=50 \ --upstream proxied-quay Added upstream reset_peer toxic 'reset_peer_upstream' on proxy 'proxied-quay' [jianl@jianl-thinkpadt14gen4 418]$ Restart OSUS instance: [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection scale deployment/sample --replicas=0 deployment.apps/sample scaled [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection scale deployment/sample --replicas=1 deployment.apps/sample scaled [jianl@jianl-thinkpadt14gen4 418]$ [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection get pods NAME READY STATUS RESTARTS AGE fault-injecting-proxy-987695595-5m89j 2/2 Running 0 170m graph-data-tag-digest 0/1 Completed 0 24m sample-8674c5d6-h4jfd 0/2 Running 0 58s [jianl@jianl-thinkpadt14gen4 418]$ [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection logs pods/sample-8674c5d6-h4jfd Still can see 502 error, but fewer than before: [2025-01-15T04:56:15Z TRACE cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] [4.10.63-x86_64] Caching release metadata [2025-01-15T04:56:15Z TRACE cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] [4.11.0-0.art-assembly-4.11.2-multi-2022-08-24-020856-x86_64] Fetching release [2025-01-15T04:56:15Z TRACE cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] [4.11.0-0.art-assembly-4.11.2-multi-2022-08-24-020856-x86_64] Processing openshift-release-dev/ocp-release [2025-01-15T04:56:15Z TRACE cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] [sha256:c46d020330f61f7617beadab052ddedc2c763f0e7c123856f96319f826c96650] Fetching release [2025-01-15T04:56:15Z TRACE cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] [sha256:c46d020330f61f7617beadab052ddedc2c763f0e7c123856f96319f826c96650] Processing openshift-release-dev/ocp-release [2025-01-15T04:56:15Z WARN cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] error fetching manifest and manifestref for openshift-release-dev/ocp-release:4.11.0-0.art-assembly-4.11.2-multi-2022-08-24-020856-ppc64le: fetching manifest and manifestref for openshift-release-dev/ocp-release:4.11.0-0.art-assembly-4.11.2-multi-2022-08-24-020856-ppc64le: unexpected HTTP status 502 Bad Gateway, ignoring this image [2025-01-15T04:56:15Z TRACE cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] [4.11.0-0.art-assembly-4.11.2-multi-2022-08-23-120030] Looking for release-manifests/release-metadata in archive sha256:34b89819d9bda04814d0fc1b618ce4660cd93fdc40dcf62cccb47735fe173a34 with 884032 bytes [2025-01-15T04:56:15Z TRACE cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] [4.11.0-0.art-assembly-rc.0-multi-2022-06-29-100843] Fetching release [2025-01-15T04:56:15Z TRACE cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] [4.11.0-0.art-assembly-rc.0-multi-2022-06-29-100843] Processing openshift-release-dev/ocp-release [2025-01-15T04:56:15Z TRACE cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] [4.11.0-0.art-assembly-4.11.2-multi-2022-08-23-120030] Caching release metadata [2025-01-15T04:56:15Z TRACE cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] [4.11.0-0.art-assembly-rc.0-multi-2022-06-29-100843-aarch64] Fetching release [2025-01-15T04:56:15Z TRACE cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] [4.11.0-0.art-assembly-rc.0-multi-2022-06-29-100843-aarch64] Processing openshift-release-dev/ocp-release [2025-01-15T04:56:15Z ERROR graph_builder::graph] failed to fetch all release metadata from fault-injecting-proxy.fault-injection.svc:443/openshift-release-dev/ocp-release [2025-01-15T04:56:15Z ERROR graph_builder::graph] fetching manifest and manifestref for openshift-release-dev/ocp-release:sha256:c46d020330f61f7617beadab052ddedc2c763f0e7c123856f96319f826c96650: unexpected HTTP status 502 Bad Gateway [jianl@jianl-thinkpadt14gen4 418]$ Replace two container's image by: [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection edit deployment/sample deployment.apps/sample edited [jianl@jianl-thinkpadt14gen4 418]$ [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection scale deployment/sample --replicas=0 deployment.apps/sample scaled [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection scale deployment/sample --replicas=1 deployment.apps/sample scaled [jianl@jianl-thinkpadt14gen4 418]$ [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection get pods NAME READY STATUS RESTARTS AGE fault-injecting-proxy-987695595-5m89j 2/2 Running 0 177m graph-data-tag-digest 0/1 Completed 0 31m sample-645df4f75d-26zpn 0/2 Running 0 20s sample-645df4f75d-h7rfz 0/2 Terminating 0 114s sample-8674c5d6-h4jfd 0/2 Terminating 2 (7s ago) 8m10s [jianl@jianl-thinkpadt14gen4 418]$ [jianl@jianl-thinkpadt14gen4 418]$ [jianl@jianl-thinkpadt14gen4 418]$ oc -n fault-injection logs pods/sample-645df4f75d-26zpn [2025-01-15T05:04:54Z TRACE cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] [4.11.0-0.nightly-multi-2022-06-20-134359-s390x] Fetching release [2025-01-15T05:04:54Z TRACE cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] [4.11.0-0.nightly-multi-2022-06-20-134359-s390x] Processing openshift-release-dev/ocp-release [2025-01-15T05:04:54Z TRACE cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] [4.11.0-0.nightly-multi-2022-06-20-130645] Downloading layer sha256:6df25de4a8b7c11f3968afb21fa115de55fae4dd680782b7d193887e00263e86 [2025-01-15T05:04:54Z WARN cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] getting manifest and manifestref failed (attempt 1/3): fetching manifest and manifestref for openshift-release-dev/ocp-release:4.11.0-0.nightly-multi-2022-06-20-134359-ppc64le: unexpected HTTP status 502 Bad Gateway, retrying... [2025-01-15T05:04:54Z TRACE cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] [4.11.0-0.nightly-multi-2022-06-20-111053] Looking for release-manifests/release-metadata in archive sha256:c81c790ea5c6e2871443426f942baff8fd40beed1af765f8cb57679be5c27025 with 873163 bytes [2025-01-15T05:04:54Z TRACE cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] [sha256:846082cac148d6e04a722fdbfcf35af180c3bde331db1bdc1bac312e499853f7] Fetching release [2025-01-15T05:04:54Z TRACE cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] [sha256:846082cac148d6e04a722fdbfcf35af180c3bde331db1bdc1bac312e499853f7] Processing openshift-release-dev/ocp-release [2025-01-15T05:04:54Z TRACE cincinnati::plugins::internal::graph_builder::release_scrape_dockerv2::registry] [4.11.0-0.nightly-multi-2022-06-20-111053] Caching release metadata