# copied from jobs/release_delivery/ospd.groovy if [[ "${IR_PATCHES_TOPIC:-}" != '' ]]; then query_url="https://review.gerrithub.io/changes/?O=10b&q=topic%3A%22${IR_PATCHES_TOPIC}%22%20status%3Aopen%20project%3A%22redhat-openstack/infrared%22" timeout 60s curl --silent --fail "$query_url" | tail -n '+2' | jq '.' | sed -nr 's/^\s*"ref": "(.*)".*/\1/p' | uniq > IR_PATCHES_TOPIC.log cat IR_PATCHES_TOPIC.log | while read REF; do timeout 60s git fetch "https://review.gerrithub.io/redhat-openstack/infrared" "$REF" git cherry-pick FETCH_HEAD done fi git log -n 5 --pretty virtualenv /tmp/ir-venv-wk3kSgh echo "export IR_HOME=`pwd`" >> /tmp/ir-venv-wk3kSgh/bin/activate . /tmp/ir-venv-wk3kSgh/bin/activate pip install -U pip pip install . > /home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/ir_pip.log pip list --format=columns > /home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/ir_pip_packages.txt cp infrared.cfg.example infrared.cfg infrared plugin add all cat << EOF > ansible.cfg [defaults] host_key_checking = False forks = 500 timeout = 30 force_color = 1 roles_path = infrared/common/roles library = infrared/common/library filter_plugins = infrared/common/filter_plugins callback_plugins = infrared/common/callback_plugins callback_whitelist = timer,profile_tasks,junit_report [ssh_connection] pipelining = True control_path = /tmp/ir-venv-wk3kSgh/%%h-%%r EOF sed -i 's/callback_whitelist.*/callback_whitelist = timer,profile_tasks,junit_report/' ansible.cfg . /tmp/ir-venv-wk3kSgh/bin/activate infrared plugin remove tripleo-upgrade infrared plugin add --revision stable/wallaby tripleo-upgrade # use mv instead of symbolic link to avoid too many levels of symbolic links issue mkdir -p $(pwd)/plugins/tripleo-upgrade/infrared_plugin/roles/tripleo-upgrade find $(pwd)/plugins/tripleo-upgrade -maxdepth 1 -mindepth 1 -not -name infrared_plugin -exec mv '{}' $(pwd)/plugins/tripleo-upgrade/infrared_plugin/roles/tripleo-upgrade \; if [ ! -f workarounds.yaml ]; then curl -sSko workarounds.yaml https://gitlab.cee.redhat.com/jbadiapa/workarounds/-/raw/cert_issue/upgrade/16.2-17.1_upgrade_workarounds.yaml fi sed -i -E 's|(.*rhos-release 17.1.*)-p [A-Za-z0-9_.-]+|\1-p passed_phase2|' workarounds.yaml sed -i -E 's|PRODUCT_BUILD=[A-Za-z0-9_.-]+|PRODUCT_BUILD=passed_phase2|' workarounds.yaml sed -i -E 's|(.*rhos-release 17.1.*)-p [A-Za-z0-9_.-]+|\1-p RHOS-17.1-RHEL-9-20230426.n.1|' workarounds.yaml sed -i -E 's|FFU_PRODUCT_BUILD=[A-Za-z0-9_.-]+|FFU_PRODUCT_BUILD=RHOS-17.1-RHEL-9-20230426.n.1|' workarounds.yaml sed -i -E 's|FFU_EL8_PRODUCT_BUILD=[A-Za-z0-9_.-]+|FFU_EL8_PRODUCT_BUILD=RHOS-17.1-RHEL-8-20230426.n.1|' workarounds.yaml sed -i -E 's|FFU_EL9_PRODUCT_BUILD=[A-Za-z0-9_.-]+|FFU_EL9_PRODUCT_BUILD=RHOS-17.1-RHEL-9-20230426.n.1|' workarounds.yaml sed -i -E 's|(.*rhos-release 17.1.*)-r [0-9.]+|\1-r 9.2|' workarounds.yaml sed -i -E 's|UPGRADE_OSP_VERSION=[0-9.]+|UPGRADE_OSP_VERSION=9.2|' workarounds.yaml . /tmp/ir-venv-wk3kSgh/bin/activate infrared virsh -o cleanup.yml --host-address osp-upgrade-01.lab.eng.brq2.redhat.com --host-key /home/rhos-ci/.ssh/rhos-jenkins/id_rsa --cleanup yes /usr/bin/python3 -m venv $WORKSPACE/.awsclientvenv source $WORKSPACE/.awsclientvenv/bin/activate python3 -m pip install --quiet awscli . /tmp/ir-venv-wk3kSgh/bin/activate infrared virsh -o provision.yml --topology-nodes undercloud:1,controller:3,computehci:3 --topology-network 3_nets_ipv4 --host-address osp-upgrade-01.lab.eng.brq2.redhat.com --host-key /home/rhos-ci/.ssh/rhos-jenkins/id_rsa --host-memory-overcommit True --image-url http://rhos-qe-mirror.lab.eng.brq2.redhat.com/rhel-8/rel-eng/RHEL-8/latest-RHEL-8.4.0/compose/BaseOS/x86_64/images/rhel-guest-image-8.4-992.x86_64.qcow2 --serial-files true --bootmode uefi --collect-ansible-facts False --virsh-snapshot-download yes --virsh-snapshot-container s3://image_sets --virsh-snapshot-path /home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/17.1-RHOS-17.1-RHEL-8-20230426.n.1-rhel-8.4-3cont_3hci-ipv4-vxlan-ovs_dvr --virsh-snapshot-import yes --virsh-snapshot-cleanup true -e override.undercloud.memory=32768 -e override.undercloud.disks.disk1.size="100G" -e override.computehci.disks.disk1.size=20G -e override.ceph.memory=6144 --topology-timezone UTC if [ '' == '' ]; then if [ -z "$(infrared plugin list | grep install-redhat-ca)" ]; then infrared plugin add "https://gitlab.cee.redhat.com/rhos-ci/ir-redhat-CA.git" fi infrared install-redhat-ca --hosts-pattern="undercloud" fi . /tmp/ir-venv-wk3kSgh/bin/activate infrared virsh -v --host-address osp-upgrade-01.lab.eng.brq2.redhat.com --host-key /home/rhos-ci/.ssh/rhos-jenkins/id_rsa --virsh-snapshot-quiesce True --ansible-args="tags=quiesce" unset IR_REGISTRY_CEPH_NAMESPACE IR_REGISTRY_CEPH_IMAGE IR_REGISTRY_CEPH_TAG . /tmp/ir-venv-wk3kSgh/bin/activate infrared tripleo-upgrade --undercloud-ffu-os-upgrade yes --upgrade-ffu-workarounds true -e @workarounds.yaml --skiplist-validations repos -e leapp_unsubscribed=True -e leapp_skip_release_check=True # #stage2('Overcloud Ctlplane System Upgrade FFU OSP17') { # . /tmp/ir-venv-wk3kSgh/bin/activate infrared tripleo-upgrade --deployment-files composable_roles --overcloud-ffu-upgrade yes --overcloud-ffu-releases '17.1,-,17.1' --upgrade-floatingip-check yes --upgrade-workload yes --overcloud-stack qe-Cloud-0 --upgrade-ffu-workarounds yes -e @workarounds.yaml --upgrade-workloadcleanup yes -e upgrade_prepare_extra_params="/home/stack/overcloud-params.yaml,/home/stack/overcloud-deploy/qe-Cloud-0/qe-Cloud-0-network-environment.yaml,/home/stack/tmp/baremetal_deployment.yaml,/home/stack/tmp/generated-networks-deployed.yaml,/home/stack/tmp/generated-vip-deployed.yaml,/usr/share/openstack-tripleo-heat-templates/environments/cephadm/cephadm-rbd-only.yaml,/usr/share/openstack-tripleo-heat-templates/environments/nova-hw-machine-type-upgrade.yaml,/home/stack/system_upgrade.yaml" -e tripleo_upgrade_debug=True --ansible-args="skip-tags=ffu_overcloud_run_compute,ffu_overcloud_upgrade_run,ffu_overcloud_ceph,ffu_overcloud_converge,ffu_overcloud_post" . /tmp/ir-venv-wk3kSgh/bin/activate infrared tripleo-upgrade --deployment-files composable_roles --overcloud-ffu-upgrade yes --overcloud-ffu-releases '17.1,-,17.1' --upgrade-floatingip-check yes --upgrade-workload yes --overcloud-stack qe-Cloud-0 --upgrade-ffu-workarounds yes -e @workarounds.yaml --upgrade-workloadcleanup yes -e upgrade_prepare_extra_params="/home/stack/overcloud-params.yaml,/home/stack/overcloud-deploy/qe-Cloud-0/qe-Cloud-0-network-environment.yaml,/home/stack/tmp/baremetal_deployment.yaml,/home/stack/tmp/generated-networks-deployed.yaml,/home/stack/tmp/generated-vip-deployed.yaml,/usr/share/openstack-tripleo-heat-templates/environments/cephadm/cephadm-rbd-only.yaml,/usr/share/openstack-tripleo-heat-templates/environments/nova-hw-machine-type-upgrade.yaml,/home/stack/system_upgrade.yaml,/home/stack/libvirt_hack.yaml" -e tripleo_upgrade_debug=True --ansible-args="skip-tags=ffu_overcloud_run_ctlplane,ffu_overcloud_ceph,ffu_overcloud_converge,ffu_overcloud_post" . /tmp/ir-venv-wk3kSgh/bin/activate infrared tempest -v -o test2.yml --setup rpm --openstack-installer tripleo --openstack-version 17.1 --image http://rhos-qe-mirror.lab.eng.brq2.redhat.com/images/cirros-0.5.2-x86_64-disk.img --openstack-installer tripleo --tests sanity,smoke,cinder_backup --dir tempest_17.1 --results-formats junitxml,html --tempest-config yes --config-options ssh_credentials.overcloud_user=tripleo-admin --threads 8 --cleanup yes --config-options service_available.mistral='false' --config-options ssh_credentials.overcloud_key_file='/home/stack/.ssh/id_extra_keys' --config-options whitebox.ctlplane_ssh_private_key_path='/home/stack/.ssh/id_extra_keys' . /tmp/ir-venv-wk3kSgh/bin/activate unset ANSIBLE_FORCE_COLOR pushd "/home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/infrared" # try exporting workspace, in case of issue need to still continue with log collection infrared workspace export -K -f workspace || true printenv | sort &> env.log pushd "/home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr" # gather all files we want from jenkins slave into collect_dir for artcl to pick them up mkdir -p /home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/logs; shopt -s globstar; cp --parents -r **/*.log /home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/logs ; cp --parents -r infrared/*.cfg /home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/logs ; cp --parents -r infrared/*.yml /home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/logs ; cp --parents -r infrared/*.yaml /home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/logs || true; cp --parents -r infrared/.workspaces/ /home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/logs ; cp --parents -r infrared/workspace.tgz /home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/logs ; cp --parents -r infrared/.envrc /home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/logs || true; cp --parents -r jump/**/*.json /home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/logs || true; pushd "/home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/infrared" infrared plugin add https://github.com/openstack/ansible-role-collect-logs.git --src-path infrared_plugin infrared plugin list export ANSIBLE_ROLES_PATH="/home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/infrared/plugins" export ANSIBLE_LOG_PATH="/home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/infrared/collect.log" # do not cause failure here see RHOSINFRA-3205 set +eo pipefail infrared ansible-role-collect-logs --disable_artifacts_cleanup true --local_working_dir '/tmp/ir-venv-wk3kSgh' --artcl_collect_dir '/home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/logs' --artcl_txt_rename true --artcl_gzip true --artcl_find_maxdepth 10 --artcl_use_rsync true --artcl_rsync_collect_list false --artcl_build_url "" --artcl_publish false --ara_enabled false --openstack_nodes all:!localhost:!hypervisor --collect_log_types container,monitoring,network,openstack,system --artcl_collect_list /etc/,/etc/neutron,/etc/tempest/*.xml,/etc/tempest/saved_state.json,/etc/tempest/tempest.conf,/etc/tempest/tempest.conf.sample,/home/*/*.conf,/home/*/*.json,/home/*/*.log,/home/*/*.sh,/home/*/*.yaml,/home/*/*.yml,/home/*/*/black_list_*,/home/*/*/white_list_*,/home/*/*rc,/home/*/.instack/install-undercloud.log,/home/*/.tripleo,/home/*/central/*.yaml,/home/*/central/*/*.yaml,/home/*/central/network/nic-configs/,/home/*/composable_roles/*.yaml,/home/*/composable_roles/*/*.yaml,/home/*/composable_roles/network/nic-configs/,/home/*/config-download/,/home/*/dcn*/*.yaml,/home/*/dcn*/*/*.yaml,/home/*/dcn*/network/nic-configs/,/home/*/inventory/group_vars/*.yml,/home/*/openshift_deploy_logs/*.log,/home/*/ostest/,/home/*/ovb,/home/*/overcloud_deploy.sh,/home/*/overcloudrc*,/home/*/playbooks_logs/*.log,/home/*/robot/,/home/*/shiftstackrc*,/home/*/tempest*/*.log,/home/*/tempest*/*.xml,/home/*/tempest*/etc/*.conf,/home/*/tempest*/saved_state.json,/home/*/tempest*/.stestr/,/home/*/tripleo-heat-installer-templates/,/home/*/undercloud-ansible-*,/home/*/undercloud-install-*.tar.bzip2,/home/*/virt,/home/*/tripleo-deploy/,/home/*/overcloud-deploy/,/home/*/templates/,/root/,/usr/share/ceph-osd-run.sh,/usr/share/openstack-tripleo-heat-templates,/var/lib/cloud/,/var/lib/config-data/,/var/lib/config-data/puppet-generated/,/var/lib/config-data/puppet-generated/neutron/etc/neutron/plugins/,/var/lib/container-puppet/,/var/lib/docker-puppet,/var/lib/heat-config,/var/lib/libvirt/qemu/*.log,/var/lib/mistral/,/var/lib/neutron/,/var/lib/openvswitch/ovn/*.db,/var/lib/tripleo-config,/var/log/,/var/log/containers/opendaylight,/var/log/extra/containers/,/var/log/extra/podman/containers,/var/run/heat-config,/var/tmp/packstack,rally-dir/*.html,rally-dir/*.log,rally-dir/*.txt,rally-dir/*.xml --artcl_exclude_list '.*',/etc/pki/*,/etc/selinux/targeted/*,/root/*.initrd*,/root/*.tar*,/root/*.vmlinuz*,/root/*.qcow*,/udev/hwdb.bin,/var/lib/config-data/*/etc/puppet/modules,/var/lib/config-data/*/etc/selinux/targeted/*,/var/log/journal/* echo "Collecting logs from hypervisor" infrared ansible-role-collect-logs --disable_artifacts_cleanup true --local_working_dir '/tmp/ir-venv-wk3kSgh' --artcl_collect_dir '/home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/logs' --artcl_txt_rename true --artcl_gzip true --artcl_find_maxdepth 10 --artcl_use_rsync true --artcl_rsync_collect_list false --artcl_build_url "" --artcl_publish false --ara_enabled false --openstack_nodes hypervisor --collect_log_types hypervisor --artcl_commands hypervisor.journal.cmd='journalctl -x --since=-8h --lines=100000',hypervisor.journal-kernel.cmd='journalctl -xk --since=-8h --lines=100000',hypervisor.virsh-list.cmd='virsh list --all',hypervisor.disk.cmd='blkid;lsblk;df -T;df -i;',hypervisor.memory.cmd='free -m',hypervisor.rpms.cmd='rpm -qa' --artcl_collect_list '/etc/ssh/,/var/lib/libvirt/qemu/*.log,/var/log/extra/journal.txt,/var/log/extra/journal-kernel.txt,/var/log/extra/virsh-list.txt,/var/log/extra/disk.txt,/var/log/extra/memory.txt,/var/log/extra/rpms.txt' artcl_exit=$? ### Find known big issues (oom,segfault,selinux) in logs ### ... count them and print as build marks pushd "/home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/logs" # do not cause failure if we have no-hits or missing files here set +eo pipefail findcat() { while read F; do if [[ "$F" =~ .gz$ ]]; then zcat "$F" else cat "$F" fi done < <(find . "$@") # all params are passed to find } # here we use sed to strip all pid/uid and such numbers in an attempt of counting # just (almost) unique avc denials (not just 8000 of repeated ones) # # (there some numbered contexts (e.g. subj=...,c96,c442) which so far seem to have no info, # still resulting in just single audit2allow entry, so stripping those too SELINUX=$(findcat -maxdepth 5 -path '*/var/log/audit/audit*' | grep -i denied | sed -r 's/(audit\(|(\S+id|ino|ses)=|[,.]c)[0-9.:]+/\1.../g' | sort -u | wc -l) OOM=$(findcat -maxdepth 4 -path '*/var/log/messages*' | grep -i oom-killer | wc -l) SEGFAULT=$(findcat -maxdepth 4 -path '*/var/log/messages*' | grep -i segfault | wc -l) set +x # split to separate strings so that presence of code in console is not matched as mark itself echo "Build" "mark: selinux_problems_found=$SELINUX" echo "Build" "mark: oom_killer_problems_found=$OOM" echo "Build" "mark: segfault_problems_found=$SEGFAULT" echo "rsyncing logs to rhos-ci@rhos-ci-logs.lab.eng.tlv2.redhat.com:/rhos-infra-dev-netapp/jenkins-logs/rcj/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/21/" du -sh /home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/logs/* # Copy console logs from all stages to the collect_dir mkdir -p /home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/logs/console_logs cp -r /home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/.sh/* /home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/logs/console_logs # Publish logs ssh rhos-ci@rhos-ci-logs.lab.eng.tlv2.redhat.com mkdir -p "/rhos-infra-dev-netapp/jenkins-logs/rcj/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/21" rsync -av --quiet /home/rhos-ci/jenkins/workspace/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/logs/* "rhos-ci@rhos-ci-logs.lab.eng.tlv2.redhat.com:/rhos-infra-dev-netapp/jenkins-logs/rcj/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/21/" echo "rsyncing logs finished" # Symlink README located in the collect server to this build log directory ssh rhos-ci@rhos-ci-logs.lab.eng.tlv2.redhat.com 'ln -s /rhos-infra/rhos-logs-readme.html /rhos-infra-dev-netapp/jenkins-logs/rcj/DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_3hci-ipv4-ovs_dvr/21/README.html' exit $artcl_exit