def stage_inside_try_post() {
/**
* Export an image set from a virsh host using infrared.
*
* @requires install_ir.groovy.inc
*
* @requires ir_virsh_cleanup_provision.groovy.inc
*
* @requires OS_CLOUD and OS_STORAGE_URL environment
* variables to be set and a clouds.yaml file
* which includes the credentials for OS_CLOUD.
*
*/
stage2('Export & Upload Image Set') {
dir('infrared') {
product_version = env.OSP_VERSION ?: "16.2"
product_build = env.PRODUCT_BUILD ?: "passed_phase2"
rhel_version = env.RHEL_VERSION ?: "rhel-8.4"
IMAGE_SET_NAME = env.IMAGE_SET_NAME ?: ""
IMAGE_SET_DOWNLOAD_PATH = env.IMAGE_SET_DOWNLOAD_PATH ?: env.WORKSPACE
if (IMAGE_SET_NAME != "") {
if (IMAGE_SET_NAME.contains("${product_build}")) {
os_version = RHEL_VERSION.replaceAll('rhel-','').replaceAll('\\..*','')
real_puddle = puddleDateFormatter(product_version, product_build, null, os_version)
IMAGE_SET_NAME = IMAGE_SET_NAME.replace("${product_build}", real_puddle)
echo "### Updated IMAGE_SET_NAME to $IMAGE_SET_NAME"
}
image_set_container = env.IMAGE_SET_CONTAINER ?: "s3://image_sets"
image_set_cleanup = env.IMAGE_SET_CLEANUP ?: True
sh2 """
. $ir_venv/bin/activate
echo "### Exporting the current running environment as an image set."
infrared virsh -v \
--host-address $host \
--host-key $HOME/.ssh/rhos-jenkins/id_rsa \
--virsh-snapshot-export yes \
--virsh-snapshot-path $IMAGE_SET_DOWNLOAD_PATH/$IMAGE_SET_NAME \
--virsh-snapshot-upload yes \
--virsh-snapshot-container $image_set_container \
--virsh-snapshot-cleanup $image_set_cleanup
echo "### Quiesce the environment after the VM's were restarted."
infrared virsh -v \
--host-address $host \
--host-key $HOME/.ssh/rhos-jenkins/id_rsa \
--virsh-snapshot-quiesce True \
--ansible-args="tags=quiesce"
"""
}
}
}
}
def stage_inside_try_2_pre() {
}
def stage_inside_try_2() {
/**
* OpenStack Undercloud Fast Forward Upgrade Content Switch.
*
* @requires install_ir.groovy.inc
*
* @param $ir_venv virtual environment where InfraRed is installed,
* by install_ir.groovy.inc stage.
*
* @jjb_param {ir_tripleo_mirror} Specified mirror
*
*/
stage2('Undercloud FFU Switch Content') {
dir('infrared') {
SELECTED_MIRROR = env.QE_MIRROR.trim() ?: ""
def IR_FFU_EXTRA_OPTIONS = ""
def IR_TRIPLEO_OS_UPGRADE_EXTRA_VARS = ""
sh2 """
. $ir_venv/bin/activate
infrared tripleo-undercloud \
-o undercloud-upgrade.yml \
--upgrade yes \
--mirror "${ SELECTED_MIRROR }" \
--build ${ env.FFU_PRODUCT_BUILD } \
--version 17.1 \
--ansible-args="tags=discover_python,upgrade_repos,undercloud_version_discovery,undercloud_containers,export"
""", basename: 'ir-tripleo-undercloud-upgrade-repo'
}
buildMarksReverseSearch([["core_puddle", "pink"]])
}
/**
* Run OSP>=17 tripleo "pre-upgrade" validations
*
* @requires install_ir.groovy.inc
*
*/
stage2('Pre-Upgrade Validations 17'){
dir('infrared') {
IR_TRIPLEO_PRE_UPGRADE_VALIDATIONS_EXTRA_VARS=''
IR_TRIPLEO_PRE_UPGRADE_VALIDATIONS_EXTRA_VARS = env.IR_TRIPLEO_PRE_UPGRADE_VALIDATIONS_EXTRA_VARS ?: """ """.trim()
if ( IR_TRIPLEO_PRE_UPGRADE_VALIDATIONS_EXTRA_VARS != '' ) {
IR_TRIPLEO_PRE_UPGRADE_VALIDATIONS_EXTRA_VARS = "--extra-vars ${IR_TRIPLEO_PRE_UPGRADE_VALIDATIONS_EXTRA_VARS}"
}
IR_TRIPLEO_OVERCLOUD_STACK_NAME = 'qe-Cloud-0' ?: "overcloud"
try {
sh2 """
. $ir_venv/bin/activate
infrared ssh `ir workspace node-list -g undercloud -f json | jq -r .nodes[].name` \\
"source stackrc && \\
sudo dnf -y update openstack-tripleo-validations python3-validations-libs validations-common && \\
sudo chown stack:stack /var/lib/mistral/.ssh/tripleo-admin-rsa && \\
sudo cat /var/lib/mistral/${IR_TRIPLEO_OVERCLOUD_STACK_NAME}/tripleo-ansible-inventory.yaml > inventory.yaml && \\
validation run -i inventory.yaml --group pre-upgrade ${IR_TRIPLEO_PRE_UPGRADE_VALIDATIONS_EXTRA_VARS}"
"""
}
catch (err) {
unstable(message: "Error detected with ${env.STAGE_NAME}, printed below")
echo err.getMessage()
echo "Continuing with the upgrade..."
}
}
}
/**
* OpenStack Undercloud Fast Forward Upgrade.
*
* @requires install_ir.groovy.inc
*
* @param $ir_venv virtual environment where InfraRed is installed,
* by install_ir.groovy.inc stage.
*
* @jjb_param {ir_tripleo_overcloud_stack_name} Custom overcloud stack name
*
* @jjb_param {ir_tripleo_upgrade_ffu_version} ffu upgrade version
*/
stage2('Undercloud FFU Upgrade 17') {
dir('infrared') {
IR_TRIPLEO_OVERCLOUD_STACK_NAME=''
if ( 'qe-Cloud-0' != '' ) {
IR_TRIPLEO_OVERCLOUD_STACK_NAME = '--overcloud-stack qe-Cloud-0'
}
IR_FFU_EXTRA_OPTIONS = ""
if (env.UPGRADE_WORKAROUNDS) {
IR_FFU_EXTRA_OPTIONS+=' --upgrade-ffu-workarounds true -e @workarounds.yaml'
}
sh2 """
unset IR_REGISTRY_CEPH_NAMESPACE IR_REGISTRY_CEPH_IMAGE IR_REGISTRY_CEPH_TAG
. $ir_venv/bin/activate
infrared tripleo-upgrade \
--undercloud-ffu-upgrade yes \
--undercloud-ffu-releases '16.2,-,17.1' \
${IR_TRIPLEO_OVERCLOUD_STACK_NAME} \
${ IR_FFU_EXTRA_OPTIONS } \
""", basename: 'ir-tripleo-undercloud-ffu-upgrade', maxLines: -1
}
buildMarksReverseSearch([["core_puddle", "pink"]])
}
/**
* OpenStack OverCloud Fast Forward Upgrade.
*
* Fast Forward Upgrade prepare step.
*
* @requires install_ir.groovy.inc
*
* @param $ir_venv virtual environment where InfraRed is installed,
* by install_ir.groovy.inc stage.
*
* @jjb_param {ir_tripleo_overcloud_stack_name} Custom overcloud stack name
*
* @jjb_param {ir_tripleo_upgrade_ffu_version} ffu upgrade version
*
* @jjb_param {ir_tripleo_overcloud_update_files} Specifies overcloud deploy files for
* updates if name different than virt
*
* @jjb_param {ir_tripleo_overcloud_deployment_files} Specifies templates
* of the overcloud deployment. By default presets are 'virsh'
* which are templates for virtual POC environment.
* Please refer to InfraRed 'tripleo-overcloud' plugin for more
* information.
*
*/
stage2('Overcloud FFU prepare') {
dir('infrared') {
IR_TRIPLEO_OVERCLOUD_STACK_PARAM=''
IR_TRIPLEO_OVERCLOUD_STACK_NAME = "overcloud"
if ( 'qe-Cloud-0' != '' ) {
IR_TRIPLEO_OVERCLOUD_STACK_NAME = 'qe-Cloud-0'
IR_TRIPLEO_OVERCLOUD_STACK_PARAM = '--overcloud-stack qe-Cloud-0'
}
IR_FFU_EXTRA_OPTIONS=''
if (env.UPGRADE_WORKAROUNDS) {
IR_FFU_EXTRA_OPTIONS+=' --upgrade-ffu-workarounds yes -e @workarounds.yaml'
}
if ( 16.2 > 13 ) {
IR_FFU_EXTRA_OPTIONS+=' --upgrade-workloadcleanup no'
}
if ( ( 16.2 >= 13 ) && ( 16.2 <= 16 ) ) {
IR_FFU_EXTRA_OPTIONS+=' -e upgrade_prepare_extra_params="/home/stack/overcloud-params.yaml"'
}
if ( 16.2 > 16 ) {
// for now we provide overcloud adoption envs in WA into ~/tmp
IR_FFU_EXTRA_OPTIONS+=" -e upgrade_prepare_extra_params=\"/home/stack/overcloud-params.yaml,/home/stack/overcloud-deploy/${IR_TRIPLEO_OVERCLOUD_STACK_NAME}/${IR_TRIPLEO_OVERCLOUD_STACK_NAME}-network-environment.yaml,/home/stack/tmp/baremetal_deployment.yaml,/home/stack/tmp/generated-networks-deployed.yaml,/home/stack/tmp/generated-vip-deployed.yaml\""
// Removal of heat-admin was introduced into 16.2.4 so we still use it just to be sure
IR_FFU_EXTRA_OPTIONS+=' --overcloud-ssh-user heat-admin'
}
if ( '' != '' ) {
IR_FFU_EXTRA_OPTIONS+=' --overcloud-ffu-replace-env-files ""'
}
IR_TRIPLEO_OVERCLOUD_DEPLOYMENT_FILES = env.OVERCLOUD_DEPLOYMENT_FILES ?: """virt """.trim()
if ('' != '') {
IR_TRIPLEO_OVERCLOUD_DEPLOYMENT_FILES = ""
}
IR_TRIPLEO_UPGRADE_VALIDATION=''
if ( '' != '' ) {
IR_TRIPLEO_UPGRADE_VALIDATION = '--run-validations '
}
IR_TRIPLEO_UPGRADE_SKIPLIST_VALIDATIONS=''
if ( '' != '' ) {
IR_TRIPLEO_UPGRADE_SKIPLIST_VALIDATIONS = '--skiplist-validations '
}
IR_TRIPLEO_UPGRADE_PREPARE_VALIDATIONS_EXTRA_ARGS=''
if ( '' != '' ) {
IR_TRIPLEO_UPGRADE_PREPARE_VALIDATIONS_EXTRA_ARGS = '--validations-extra-args ""'
}
IR_FURIOUS_UPGRADE_OPTIONS=''
if (env.FURIOUS_UPGRADE == 'true') {
IR_FURIOUS_UPGRADE_OPTIONS=' --config-heat AllInOneUpgrade=true '
}
sh2 """
. $ir_venv/bin/activate
infrared tripleo-upgrade \
--deployment-files ${IR_TRIPLEO_OVERCLOUD_DEPLOYMENT_FILES} \
--overcloud-ffu-upgrade yes \
--overcloud-ffu-releases '16.2,-,17.1' \
--upgrade-floatingip-check no \
--upgrade-workload no \
${IR_TRIPLEO_OVERCLOUD_STACK_PARAM} \
${IR_FFU_EXTRA_OPTIONS} \
${ IR_TRIPLEO_UPGRADE_VALIDATION } \
${ IR_TRIPLEO_UPGRADE_SKIPLIST_VALIDATIONS } \
${ IR_TRIPLEO_UPGRADE_PREPARE_VALIDATIONS_EXTRA_ARGS } \
--ansible-args="skip-tags=create_ffu_scripts,ffu_overcloud_run,ffu_overcloud_upgrade_role,ffu_overcloud_ceph,ffu_overcloud_converge,ffu_overcloud_post" \
${IR_FURIOUS_UPGRADE_OPTIONS} \
\
""", basename: 'ir-tripleo-ffu-prepare', maxLines: -1
}
}
/**
* OpenStack OverCloud Fast Forward Upgrade.
*
* Run the FFU steps for upgrading Ceph
*
* @requires install_ir.groovy.inc
*
* @param $ir_venv virtual environment where InfraRed is installed,
* by install_ir.groovy.inc stage.
*
* @jjb_param {ir_tripleo_overcloud_stack_name} Custom overcloud stack name
*
* @jjb_param {ir_tripleo_overcloud_update_files} Specifies overcloud deploy files for
* updates if name different than virt
*
* @jjb_param {ir_tripleo_upgrade_ffu_version} ffu upgrade version
*
* @jjb_param {ir_tripleo_overcloud_deployment_files} Specifies templates
* of the overcloud deployment. By default presets are 'virsh'
* which are templates for virtual POC environment.
* Please refer to InfraRed 'tripleo-overcloud' plugin for more
* information.
*
*/
stage2('Overcloud FFU Ceph adopt') {
dir('infrared') {
IR_TRIPLEO_OVERCLOUD_STACK_PARAM=''
IR_TRIPLEO_OVERCLOUD_STACK_NAME = "overcloud"
if ( 'qe-Cloud-0' != '' ) {
IR_TRIPLEO_OVERCLOUD_STACK_NAME = 'qe-Cloud-0'
IR_TRIPLEO_OVERCLOUD_STACK_PARAM = '--overcloud-stack qe-Cloud-0'
}
IR_FFU_EXTRA_OPTIONS+=' --upgrade-workloadcleanup no'
// for now we provide overcloud adoption envs in WA into ~/tmp
IR_FFU_EXTRA_OPTIONS+=" -e upgrade_prepare_extra_params=\"/home/stack/overcloud-params.yaml,/home/stack/overcloud-deploy/${IR_TRIPLEO_OVERCLOUD_STACK_NAME}/${IR_TRIPLEO_OVERCLOUD_STACK_NAME}-network-environment.yaml,/home/stack/tmp/baremetal_deployment.yaml,/home/stack/tmp/generated-networks-deployed.yaml,/home/stack/tmp/generated-vip-deployed.yaml,/usr/share/openstack-tripleo-heat-templates/environments/cephadm/cephadm-rbd-only.yaml,/usr/share/openstack-tripleo-heat-templates/environments/nova-hw-machine-type-upgrade.yaml\" -e \"ceph_upgrade_skip_tags=ceph_health,opendev-validation,ceph_ansible_remote_tmp\""
IR_TRIPLEO_OVERCLOUD_DEPLOYMENT_FILES = env.OVERCLOUD_DEPLOYMENT_FILES ?: """virt """.trim()
if ('' != '') {
IR_TRIPLEO_OVERCLOUD_DEPLOYMENT_FILES = ""
}
sh2 """
. $ir_venv/bin/activate
infrared tripleo-upgrade \
--deployment-files ${IR_TRIPLEO_OVERCLOUD_DEPLOYMENT_FILES} \
--overcloud-ffu-upgrade yes \
--overcloud-ffu-releases '16.2,-,17.1' \
--upgrade-floatingip-check no \
--upgrade-workload no \
${IR_TRIPLEO_OVERCLOUD_STACK_PARAM} \
${IR_FFU_EXTRA_OPTIONS} \
--ansible-args="skip-tags=create_ffu_scripts,ffu_overcloud_prepare,ffu_overcloud_run,ffu_overcloud_upgrade_role,ffu_overcloud_converge,ffu_overcloud_post" \
infrared ssh undercloud-0 'sh overcloud_upgrade_prepare.sh'
""", basename: 'ir-tripleo-ffu-ceph', maxLines: -1
}
}
/**
* OpenStack OverCloud Fast Forward Upgrade for OSP17.
*
* Run the FFU steps for upgrading the controllers and controlplane nodes.
*
* @requires install_ir.groovy.inc
*
* @param $ir_venv virtual environment where InfraRed is installed,
* by install_ir.groovy.inc stage.
*
* @jjb_param {ir_tripleo_overcloud_stack_name} Custom overcloud stack name
*
* @jjb_param {ir_tripleo_upgrade_ffu_version} ffu upgrade version
*/
stage2('Overcloud FFU') {
timeout(time: "600".toInteger(), unit: 'MINUTES') {
dir('infrared') {
def FLOATING_IP_CHECK = "no"
IR_TRIPLEO_OVERCLOUD_STACK_PARAM=''
IR_TRIPLEO_OVERCLOUD_STACK_NAME = "overcloud"
if ( 'qe-Cloud-0' != '' ) {
IR_TRIPLEO_OVERCLOUD_STACK_NAME = 'qe-Cloud-0'
IR_TRIPLEO_OVERCLOUD_STACK_PARAM = '--overcloud-stack qe-Cloud-0'
}
IR_FFU_EXTRA_OPTIONS=''
if (env.UPGRADE_WORKAROUNDS) {
IR_FFU_EXTRA_OPTIONS+=' --upgrade-ffu-workarounds yes -e @workarounds.yaml'
}
IR_FFU_EXTRA_OPTIONS+=' --upgrade-workloadcleanup no'
// for now we provide overcloud adoption envs in WA into ~/tmp
IR_FFU_EXTRA_OPTIONS+=" -e upgrade_prepare_extra_params=\"/home/stack/overcloud-params.yaml,/home/stack/overcloud-deploy/${IR_TRIPLEO_OVERCLOUD_STACK_NAME}/${IR_TRIPLEO_OVERCLOUD_STACK_NAME}-network-environment.yaml,/home/stack/tmp/baremetal_deployment.yaml,/home/stack/tmp/generated-networks-deployed.yaml,/home/stack/tmp/generated-vip-deployed.yaml,/usr/share/openstack-tripleo-heat-templates/environments/cephadm/cephadm-rbd-only.yaml,/usr/share/openstack-tripleo-heat-templates/environments/nova-hw-machine-type-upgrade.yaml\" -e tripleo_upgrade_debug=True "
IR_FFU_SKIP_TAGS = '--ansible-args="skip-tags=create_ffu_scripts,ffu_overcloud_prepare,ffu_overcloud_upgrade_compute,ffu_overcloud_ceph,ffu_overcloud_converge,ffu_overcloud_post,ffu_overcloud_system_upgrade"'
IR_TRIPLEO_OVERCLOUD_DEPLOYMENT_FILES = env.OVERCLOUD_DEPLOYMENT_FILES ?: """virt """.trim()
if ('' != '') {
IR_TRIPLEO_OVERCLOUD_DEPLOYMENT_FILES = ""
}
IR_FFU_EXTRA_OPTIONS+=' --fast-and-furious true'
sh2 """
. $ir_venv/bin/activate
infrared tripleo-upgrade \
--deployment-files ${IR_TRIPLEO_OVERCLOUD_DEPLOYMENT_FILES} \
--overcloud-ffu-upgrade yes \
--overcloud-ffu-releases '16.2,-,17.1' \
--upgrade-floatingip-check ${FLOATING_IP_CHECK} \
--upgrade-workload no \
${IR_TRIPLEO_OVERCLOUD_STACK_PARAM} \
${IR_FFU_EXTRA_OPTIONS} \
${IR_FFU_SKIP_TAGS} \
""", basename: 'ir-tripleo-ffu-controller', maxLines: -1
}
}
}
/**
* Run OSP>=16.x tripleo "post-upgrade" validations
*
* @requires install_ir.groovy.inc
*
*/
stage2('Post-Upgrade Validations'){
dir('infrared') {
IR_TRIPLEO_POST_UPGRADE_VALIDATIONS_EXTRA_VARS=''
IR_TRIPLEO_POST_UPGRADE_VALIDATIONS_EXTRA_VARS = env.IR_TRIPLEO_POST_UPGRADE_VALIDATIONS_EXTRA_VARS ?: """ """.trim()
if ( IR_TRIPLEO_POST_UPGRADE_VALIDATIONS_EXTRA_VARS != '' ) {
IR_TRIPLEO_POST_UPGRADE_VALIDATIONS_EXTRA_VARS = "--extra-vars ${IR_TRIPLEO_POST_UPGRADE_VALIDATIONS_EXTRA_VARS}"
}
IR_TRIPLEO_OVERCLOUD_STACK_NAME=''
if ( 'qe-Cloud-0' != '' ) {
IR_TRIPLEO_OVERCLOUD_STACK_NAME = '--stack qe-Cloud-0'
}
try {
if ( ("16.2" as Float).intValue() == 16 ) {
sh2 """
. $ir_venv/bin/activate
infrared ssh `ir workspace node-list -g undercloud -f json | jq -r .nodes[].name` \\
sudo cat /var/lib/mistral/${IR_TRIPLEO_OVERCLOUD_STACK_NAME}/tripleo-ansible-inventory.yaml > inventory.yaml && \\
validation run -i inventory.yaml --group post-upgrade ${IR_TRIPLEO_PRE_UPGRADE_VALIDATIONS_EXTRA_VARS}
"""
} else if ( ("16.2" as Float).intValue() == 17 ) {
sh2 """
. $ir_venv/bin/activate
infrared ssh `ir workspace node-list -g undercloud -f json | jq -r .nodes[].name` \\
validation run -i overcloud-deploy/${IR_TRIPLEO_OVERCLOUD_STACK_NAME}/tripleo-ansible-inventory.yaml --group post-upgrade ${IR_TRIPLEO_PRE_UPGRADE_VALIDATIONS_EXTRA_VARS}
"""
} else {
sh2 """
. $ir_venv/bin/activate
infrared ssh `ir workspace node-list -g undercloud -f json | jq -r .nodes[].name` \\
"source stackrc && \\
openstack tripleo validator run \
--group post-upgrade ${IR_TRIPLEO_OVERCLOUD_STACK_NAME} \
${IR_TRIPLEO_POST_UPGRADE_VALIDATIONS_EXTRA_VARS}"
"""
}
}
catch (err) {
unstable(message: "Error detected with ${env.STAGE_NAME}, printed below")
echo err.getMessage()
echo "Continuing with the execution..."
}
}
}
}
def stage_inside_try_2_post() {
/**
* Run functional tests for stf client-side
*
* @requires install_ir.groovy.inc
*
* @param $ir_venv virtual environment where InfraRed is installed,
* by install_ir.groovy.inc stage.
*
*/
stage2('Run STF functional tests') {
if ('performance,cinder_backup_ap,custom-stf-ceph,stf-connectors-osp17.0,gnocchi-connectors-rbd,ceph-single-host-mode,barbican'.contains('stf')) {
dir('infrared') {
dir('playbooks'){
git branch: "master", url: 'https://github.com/infrawatch/functional-tests.git'
sh(script:"curl --insecure https://gitlab.cee.redhat.com/cee_ops/quicklab/raw/master/docs/quicklab.key --output quicklab.key;chmod 600 quicklab.key",returnStdout: false)
}
sh2 basename: 'stf-tests', script: """
. $ir_venv/bin/activate
ANSIBLE_CALLBACK_WHITELIST=custom_logger ansible-playbook -i `ir workspace inventory` -i playbooks/default.inv playbooks/stf_functional_tests.yml --tags performance,cinder_backup_ap,custom-stf-ceph,stf-connectors-osp17.0,gnocchi-connectors-rbd,ceph-single-host-mode,barbican,OSP16.2
ansible localhost -vvv \
-m copy -a "src=\$WORKSPACE/infrared/test_run_result.out \
dest=\$WORKSPACE/"
"""
}
archiveArtifacts artifacts: 'test_run_result.out'
currentBuild.description = (currentBuild.description ?: '') + "
\nSTF Test Results\n"
} else {
println('ir_tripleo_overcloud_templates does not contain stf, skipping stage Run STF functional tests')
}
}
/**
* Prepare to run novajoin tempest tests on the undercloud
*
* @requires install_ir.groovy.inc
*
* @param $ir_venv virtual environment where InfraRed is installed,
* by install_ir.groovy.inc stage.
*
*/
stage2('Prepare to run novajoin tempest tests') {
dir('infrared') {
if ( '' == 'novajoin' || 'sanity,smoke,cinder_backup,barbican,identity' == 'novajoin' ) {
sh2 basename: 'python-novajoin-tempest-test', script: """
. $ir_venv/bin/activate
curl -k https://gitlab.cee.redhat.com/OSP-DFG-security/automation/raw/master/playbooks/python-novajoin_tempest_prep.yml -O
ansible-playbook -i `ir workspace inventory` python-novajoin_tempest_prep.yml
"""
if ( 16.2 < 17 ) {
sh2 basename: 'python-novajoin-tempest-test-rcfile1', script: """
. $ir_venv/bin/activate
myworkspace=`dirname \$(ir workspace inventory)`
cp \$myworkspace/stackrc /tmp/novajoin_stackrc
"""
}
} else {
println("ir_tempest_tests not set to novajoin skipping stage")
}
}
}
/**
* Runs Tempest tests using InfraRed.
*
* @requires install_ir.groovy.inc
*
* @jjb_param sanity,smoke,cinder_backup,barbican,identity list of tests to be executed. This is only
* applicable if infrared tester is selected.
* Please refer to InfraRed 'tempest' plugin for more information.
*
* @jjb_param tripleo Installer that was used to
* provision OpenStack. Please refer to InfraRed 'tempest' plugin
* for more information.
*
* @jjb_param {ir_tempest_second_guest_image} optional
* Guest image to use for this stage.
*
* @jjb_param {ir_tempest_second_run_override_options} optional
* override options that can be specified by any job definition,
* such as multiple '--config-options'
*
* @jjb_param {ir_tempest_second_tests} optional
* override options that can be specified by any job definition,
* such as multiple '--config-options'
*
* @jjb_param {ir_tempest_second_config} optional
* create tempest configuration file
*
* @jjb_param if applicable (git source only)
* specifies git revision to be used for Tempest Test Suite
*
* @param $ir_venv virtual environment where InfraRed is installed,
* by install_ir.groovy.inc stage.
*
*/
stage2('Second Tempest Run') {
def FLV_IJECT = ""
TEMPEST_IMAGE_OPT = ""
TEMPEST_GUEST_IMAGE = "cirros" ?: "cirros"
OPENSTACK_VERSION = "17.1" ?: "16.2"
if ( TEMPEST_GUEST_IMAGE.contains("rhel") || TEMPEST_GUEST_IMAGE.contains("windows") ) {
def ir_volume_size = "" ?: "10"
def overcloud_rc = 'overcloudrc'
if ( '' != '' ) {
overcloud_rc = ''
} else if ( 'qe-Cloud-0' != '' ) {
overcloud_rc = 'qe-Cloud-0rc'
}
def tempest_img_list = [
'rhel-9.2':'http://download.devel.redhat.com/rhel-9/rel-eng/RHEL-9/RHEL-9.2.0-Beta-1.0/compose/BaseOS/x86_64/images/rhel-guest-image-9.2-20230306.4.x86_64.qcow2',
'rhel-9.1':'http://download.devel.redhat.com/rhel-9/rel-eng/RHEL-9/RHEL-9.1.0-20221027.3/compose/BaseOS/x86_64/images/rhel-guest-image-9.1-20221027.3.x86_64.qcow2',
'rhel-9.0':'http://download.eng.tlv.redhat.com/rhel-9/rel-eng/RHEL-9/latest-RHEL-9.0.0/compose/BaseOS/x86_64/images/rhel-guest-image-9.0-20220420.0.x86_64.qcow2',
'rhel-8.5':'http://download.eng.tlv.redhat.com/rhel-8/rel-eng/RHEL-8/latest-RHEL-8.5.0/compose/BaseOS/x86_64/images/rhel-guest-image-8.5-1174.x86_64.qcow2',
'rhel-8.4':'http://download.devel.redhat.com/rhel-8/rel-eng/RHEL-8/latest-RHEL-8.4.0/compose/BaseOS/x86_64/images/rhel-guest-image-8.4-992.x86_64.qcow2',
'rhel-8.3':'http://download.devel.redhat.com/rhel-8/rel-eng/RHEL-8/latest-RHEL-8.3.0/compose/BaseOS/x86_64/images/rhel-guest-image-8.3-401.x86_64.qcow2',
'rhel-8.2':'http://download.devel.redhat.com/rhel-8/rel-eng/RHEL-8/latest-RHEL-8.2.0/compose/BaseOS/x86_64/images/rhel-guest-image-8.2-290.x86_64.qcow2',
'rhel-8.1':'http://download.devel.redhat.com/rhel-8/rel-eng/RHEL-8/latest-RHEL-8.1.0/compose/BaseOS/x86_64/images/rhel-guest-image-8.1-263.x86_64.qcow2',
'rhel-8.0':'http://download.devel.redhat.com/rhel-8/rel-eng/RHEL-8/latest-RHEL-8.0.0/compose/BaseOS/x86_64/images/rhel-guest-image-8.0-1854.x86_64.qcow2',
'rhel-7.9':'http://download.devel.redhat.com/rhel-7/rel-eng/RHEL-7/latest-RHEL-7.9/compose/Server/x86_64/images/rhel-guest-image-7.9-30.x86_64.qcow2',
'rhel-7.8':'http://download.devel.redhat.com/rhel-7/rel-eng/RHEL-7/latest-RHEL-7.8/compose/Server/x86_64/images/rhel-guest-image-7.8-41.x86_64.qcow2',
'rhel-7.7':'http://download.devel.redhat.com/rhel-7/rel-eng/RHEL-7/latest-RHEL-7.7/compose/Server/x86_64/images/rhel-guest-image-7.7-261.x86_64.qcow2',
'rhel-7.6':'http://download.devel.redhat.com/rhel-7/rel-eng/RHEL-7/latest-RHEL-7.6/compose/Server/x86_64/images/rhel-guest-image-7.6-210.x86_64.qcow2',
'rhel-7.5':'http://download.devel.redhat.com/rhel-7/rel-eng/RHEL-7/RHEL-7.5-RC-1.3/compose/Server/x86_64/images/rhel-guest-image-7.5-146.x86_64.qcow2',
'rhel-7.4':'http://download.devel.redhat.com/rhel-7/rel-eng/RHEL-7/RHEL-7.4-RC-1.2/compose/Server/x86_64/images/rhel-guest-image-7.4-191.x86_64.qcow2',
'rhel-7.3':'http://download.devel.redhat.com/pub/rhel/released/RHEL-7/7.3/Server/x86_64/images/rhel-guest-image-7.3-33.x86_64.qcow2',
'windows2019':'http://10.0.152.55/dfg-compute-images/win2019.qcow2',
'rhel-8.3-ppc64le':'http://download.devel.redhat.com/rhel-8/rel-eng/RHEL-8/latest-RHEL-8.3.0/compose/BaseOS/ppc64le/images/rhel-guest-image-8.3-400.ppc64le.qcow2',
]
def tempest_guest_image_url = tempest_img_list[TEMPEST_GUEST_IMAGE] ?: "http://download.devel.redhat.com/rhel-8/rel-eng/RHEL-8/latest-RHEL-8.3.0/compose/BaseOS/x86_64/images/rhel-guest-image-8.3-401.x86_64.qcow2"
dir('infrared') {
// Create two flavors:
// 1. Main testing flavor
// 2. Secondary flavor ('alt') for resize tests (otherwise we get 'When resizing, instances must change flavor!')
// * First, deleting previous flavors with same ID, otherwise, the flavors creation commands will be failed when running this stage multiple times
// TODO: Move the flavors creation part to InfraRed tempest plugin
sh """
. $ir_venv/bin/activate
ansible -o -i `infrared workspace inventory` undercloud -m shell -a "source ~/${overcloud_rc} && (openstack flavor delete 200 || true) && openstack flavor create --id 200 --ram 2048 --disk ${ir_volume_size} --vcpus 2 guest_image"
ansible -o -i `infrared workspace inventory` undercloud -m shell -a "source ~/${overcloud_rc} && (openstack flavor delete 201 || true) && openstack flavor create --id 201 --ram 2048 --disk ${ir_volume_size} --vcpus 2 guest_image_alt"
"""
// NOTE: 'volume_size' needs to be bigger than default 1GB when using RHEL/Windows guest
// otherwise 'volume' tempest tests fail with:
// "If Image virtual size is 10GB and doesn't fit in a volume of size 1GB"
FLV_IJECT="--config-options compute.flavor_ref=200 --config-options compute.flavor_ref_alt=201 --config-options compute.image_ssh_user=cloud-user --config-options validation.image_ssh_user=cloud-user --config-options scenario.dhcp_client=dhclient --config-options volume.volume_size=${ir_volume_size}"
TEMPEST_IMAGE_OPT="--image ${tempest_guest_image_url}"
}
if ( TEMPEST_GUEST_IMAGE == 'rhel' ) {
IR_VIRSH_IMAGE=sh(returnStdout: true, script: 'echo ${IR_VIRSH_IMAGE:-}').trim()
TEMPEST_IMAGE_OPT="--image ${IR_VIRSH_IMAGE}"
}
} else if ( TEMPEST_GUEST_IMAGE == 'cirros' ) {
def cirros_img_url = siteURL('http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img')
def cirros_alt_img_url = siteURL('http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-uec.tar.gz')
// There is known issue with cirros-0.4 and OSP17 env - see https://issues.redhat.com/browse/RHOSINFRA-4643
if ( OPENSTACK_VERSION.toDouble() >= 17 ) {
cirros_alt_img_url = siteURL('http://rhos-qe-mirror.lab.eng.tlv2.redhat.com/images/cirros-0.5.2-x86_64-uec.tar.gz')
cirros_img_url = siteURL('http://rhos-qe-mirror.lab.eng.tlv2.redhat.com/images/cirros-0.5.2-x86_64-disk.img')
}
if ( OPENSTACK_VERSION.toDouble() <= 10 ){
TEMPEST_IMAGE_OPT = "--image " + cirros_img_url + " --config-options image.http_image=\'" + cirros_alt_img_url + "\'"
} else {
TEMPEST_IMAGE_OPT = "--image " + cirros_img_url
}
}
TEMPEST_SECOND_RUN_OVERRIDE_OPTIONS = env.TEMPEST_SECOND_RUN_OVERRIDE_OPTIONS ?: siteURL("""--threads 8 --cleanup yes \
--config-options service_available.mistral='false' """).trim()
if ( 'junitxml,html' != '' ) {
RESULTS_FORMATS='--results-formats junitxml,html'
}
// add the extra keys generated during "Post tasks" stage when OSP release
// is 17 or later - these keys are needed to connect from the undercloud to
// the overcloud nodes using paramiko
if (OPENSTACK_VERSION.toDouble() >= 17 && !TEMPEST_SECOND_RUN_OVERRIDE_OPTIONS.contains("/home/stack/.ssh/id_extra_keys")) {
TEMPEST_SECOND_RUN_OVERRIDE_OPTIONS += " --config-options ssh_credentials.overcloud_key_file='/home/stack/.ssh/id_extra_keys' --config-options whitebox.ctlplane_ssh_private_key_path='/home/stack/.ssh/id_extra_keys'"
}
// This is needed until the downstream tempest plugins switch to tripleo-admin on 17 (by branching)
if (OPENSTACK_VERSION.toDouble() >= 17) {
// Note: order is important, this allows to override overcloud admin user name using TEMPEST_SECOND_RUN_OVERRIDE_OPTIONS when it's needed
TEMPEST_SECOND_RUN_OVERRIDE_OPTIONS = "--config-options ssh_credentials.overcloud_user=tripleo-admin " + TEMPEST_SECOND_RUN_OVERRIDE_OPTIONS
}
TEMPEST_TESTS = 'sanity,smoke,cinder_backup,barbican,identity'
if ( '' != '' ) {
TEMPEST_TESTS = ''
}
TEMPEST_CONFIGURATION = ''
if ( 'yes' != '' ) {
TEMPEST_CONFIGURATION = '--tempest-config yes'
}
TEMPEST_REVISION = ''
if ( '' != '' ) {
TEMPEST_REVISION = '--revision '
}
TEMPEST_DIR = '--dir tempest-dir'
if ( '17.1' != '' ) {
TEMPEST_DIR = '--dir tempest_17.1'
}
if ("false".toBoolean()) {
log "Squashing double curly braces in Tempest override options", level: "DEBUG"
TEMPEST_SECOND_RUN_OVERRIDE_OPTIONS =
TEMPEST_SECOND_RUN_OVERRIDE_OPTIONS.replaceAll("\\{\\{", "\\{").replaceAll("\\}\\}", "\\}")
}
dir('infrared') {
sh2 """
. $ir_venv/bin/activate
infrared tempest -v \
-o test2.yml \
--setup rpm \
--openstack-installer tripleo \
--openstack-version ${OPENSTACK_VERSION} \
${TEMPEST_IMAGE_OPT} \
--openstack-installer tripleo $FLV_IJECT \
--tests ${TEMPEST_TESTS} \
${TEMPEST_DIR} \
${RESULTS_FORMATS} \
${TEMPEST_CONFIGURATION} \
${TEMPEST_REVISION} \
${TEMPEST_SECOND_RUN_OVERRIDE_OPTIONS}
""", basename: 'ir-tempest-2', echoScript: true
env.IR_TEMPEST_RUN_FINISHED = 'True'
}
}
/**
* Export an image set from a virsh host using infrared.
*
* @requires install_ir.groovy.inc
*
* @requires ir_virsh_cleanup_provision.groovy.inc
*
* @requires OS_CLOUD and OS_STORAGE_URL environment
* variables to be set and a clouds.yaml file
* which includes the credentials for OS_CLOUD.
*
*/
stage2('FFU Export & Upload Image Set') {
dir('infrared') {
FFU_PRODUCT_BUILD = env.FFU_PRODUCT_BUILD ?: "passed_phase2"
SECOND_IMAGE_SET_NAME = env.SECOND_IMAGE_SET_NAME ?: ""
IMAGE_SET_DOWNLOAD_PATH = env.IMAGE_SET_DOWNLOAD_PATH ?: env.WORKSPACE
if (SECOND_IMAGE_SET_NAME != "") {
image_set_exists=false
image_set_upload_stuck=false
image_set_complete=false
// Here we check if original job config was used eg second image set contains default product build string
if (SECOND_IMAGE_SET_NAME.contains("passed_phase2")) {
os_version = "8.4".replaceAll('rhel-','').replaceAll('\\..*','')
real_puddle = puddleDateFormatter("17.1", FFU_PRODUCT_BUILD, null, os_version)
SECOND_IMAGE_SET_NAME = SECOND_IMAGE_SET_NAME.replace("passed_phase2", real_puddle)
env.SECOND_IMAGE_SET_NAME = SECOND_IMAGE_SET_NAME
echo "### Updated SECOND_IMAGE_SET_NAME to $SECOND_IMAGE_SET_NAME"
}
// detect whether an image set exists (but may not be complete)
image_set_exists = sh (
script: """\
\$WORKSPACE/.awsclientvenv/bin/aws --endpoint-url \$AWS_ENDPOINT_URL s3 ls $image_set_container/$SECOND_IMAGE_SET_NAME/status &>/dev/null
""",
returnStatus: true
) == 0
// detect whether an image set exists and is complete
image_set_complete = sh (
script: """\
\$WORKSPACE/.awsclientvenv/bin/aws --endpoint-url \$AWS_ENDPOINT_URL s3 cp $image_set_container/$SECOND_IMAGE_SET_NAME/status - | grep -q upload-complete &>/dev/null
""",
returnStatus: true
) == 0
if ( image_set_exists && !image_set_complete ) {
image_set_upload_date = sh (
script: """\
\$WORKSPACE/.awsclientvenv/bin/aws --endpoint-url \$AWS_ENDPOINT_URL s3 ls $image_set_container/$SECOND_IMAGE_SET_NAME/status | awk '{print "\$1 \$2"}' | tr '-' '/'
""",
returnStatus: true
) == 0
date_now = new Date();
date_upload = new Date(image_set_upload_date)
upload_date_diff = (date_now.getTime()-date_upload.getTime())
echo "### Image set is uploading for ${upload_date_diff}"
image_set_upload_stuck = upload_date_diff>43200000
}
if ( !image_set_exists || image_set_upload_stuck) {
image_set_container = env.IMAGE_SET_CONTAINER ?: "s3://image_sets"
image_set_cleanup = env.IMAGE_SET_CLEANUP ?: True
sh2 """
. $ir_venv/bin/activate
echo "### Exporting the current running environment as an image set."
infrared virsh -v \
--host-address $host \
--host-key $HOME/.ssh/rhos-jenkins/id_rsa \
--virsh-snapshot-export yes \
--virsh-snapshot-path $IMAGE_SET_DOWNLOAD_PATH/$SECOND_IMAGE_SET_NAME \
--virsh-snapshot-upload yes \
--virsh-snapshot-container $image_set_container \
--virsh-snapshot-cleanup $image_set_cleanup
echo "### Quiesce the environment after the VM's were restarted."
infrared virsh -v \
--host-address $host \
--host-key $HOME/.ssh/rhos-jenkins/id_rsa \
--virsh-snapshot-quiesce True \
--ansible-args="tags=quiesce"
"""
}
}
}
}
// Triggers a follow-on job to run
timeout(time: 10, unit: 'MINUTES') {
stage2('FFU Trigger System Upgrade job') {
if ((env.NEXT_JOB_NAME) && (currentBuild.currentResult == 'SUCCESS' || currentBuild.currentResult == 'UNSTABLE')) {
// define the override parameters
def params = [
string(name: "IR_GERRIT_CHANGE", value: (env.IR_GERRIT_CHANGE ?: "").toString()),
string(name: "NODE_NAME", value: (env.NODE_NAME ?: "dfg-upgrades").toString()),
string(name: "FFU_PRODUCT_BUILD", value: (env.FFU_17_EL9_BUILD ?: "passed_phase2").toString()),
string(name: "OOO_UPGRADE_PLUGIN_GERRIT_CHANGE", value: (env.OOO_UPGRADE_PLUGIN_GERRIT_CHANGE ?: "").toString()),
string(name: "PRODUCT_BUILD", value: (env.PRODUCT_BUILD ?: "latest-RHOS-17.1-RHEL-8.4").toString()),
booleanParam(name: "PUBLISH_TO_POLARION", value: (env.PUBLISH_TO_POLARION ?: "false").toBoolean()),
string(name: "UPGRADE_RHEL_VERSION", value: "9.2"),
string(name: "UPGRADE_WORKAROUNDS", value: (env.UPGRADE_WORKAROUNDS ?: "https://gitlab.cee.redhat.com/rhos-upgrades/workarounds/-/raw/master/upgrade/16.2-17.1_upgrade_workarounds.yaml").toString()),
string(name: "IMAGE_SET_NAME", value: (env.SECOND_IMAGE_SET_NAME).toString()),
]
// trigger the next job
build job: env.NEXT_JOB_NAME, propagate: false, wait: false, parameters: params
}
}
}
}
NExt job to upgrade os :
DFG-upgrades-ffu-17.1-9.2-passed_phase2-from-17.1-rhel-8.4-latest-RHOS-17.1-RHEL-8.4-3cont_2comp_3ceph-ipv4-stf