OCM_ENV=$API_HOST SET_ENVIRONMENT="1" OC_HOST=$(oc whoami --show-server) CLUSTER_ID=$(cat "${SHARED_DIR}/cluster-id") CLUSTER_NAME=$(cat "${SHARED_DIR}/cluster-name") OCM_TOKEN=$(cat /var/run/secrets/ci.openshift.io/cluster-profile/ocm-token) ROBOT_EXTRA_ARGS="-i $TEST_MARKER -e AutomationBug -e Resources-GPU -e Resources-2GPUS" RUN_SCRIPT_ARGS="--skip-oclogin true --set-urls-variables true --test-artifact-dir ${ARTIFACT_DIR}/results" export OCM_ENV export SET_ENVIRONMENT export OC_HOST export CLUSTER_NAME export OCM_TOKEN export ROBOT_EXTRA_ARGS export RUN_SCRIPT_ARGS mkdir $ARTIFACT_DIR/results # delete IDPs before running testsuite if [ "${API_HOST}" = "stage" ]; then API_URL=https://api.stage.openshift.com/ else API_URL=https://api.openshift.com/ fi ocm login --url=$API_URL --token=$OCM_TOKEN for IDP in $(ocm get /api/clusters_mgmt/v1/clusters/$CLUSTER_ID/identity_providers | jq -r '.items[].name'); do ocm delete idp $IDP --cluster=$CLUSTER_ID done Deleted identity provider 'rosa-htpasswd' on cluster '25l8qll634ftj2o7o6proa46vke00pml' # running RHODS testsuite ./ods_ci/build/run.sh RUN SCRIPT ARGS: --skip-oclogin true --set-urls-variables true --test-artifact-dir /logs/artifacts/results ROBOT EXTRA ARGS: -i Smoke -e AutomationBug -e Resources-GPU -e Resources-2GPUS SET TEST ENVIRONMENT: 1 -- USE OCM to install IDPs: 1 -----| SET_ENVIRONMENT option is enabled. ODS-CI is going to configure the test environment for you..|----- ---> Performing log in OCM ocm envirnment: stage ---> Looking for LDAP and HTPASSWD already present in the cluster... 25l8qll634ftj2o7o6proa46vke00pml ---> Installing the required IDPs https://api.ci-rosa-s-a40b.bwgt.s1.devshift.org:6443 Cluster name is ci-rosa-s-a40b Configuring IDP for cluster 'ci-rosa-s-a40b' Identity Provider 'htpasswd' has been created. You need to ensure that there is a list of cluster administrators defined. See 'ocm create user --help' for more information. To login into the console, open https://console-openshift-console.apps.ci-rosa-s-a40b.bwgt.s1.devshift.org and click on htpasswd. Securely store your username and password. If you lose these credentials, you will have to delete and recreate the IDP. Added 'cluster-admins' user 'htpasswd-user' to cluster 'ci-rosa-s-a40b' ----> Performing log in the cluster using oc CLI command Login failed (401 Unauthorized) Verify you have provided the correct credentials. Login failed 1 times. Trying again in 30 seconds (max 7 times). Login successful. You have access to 115 projects, the list has been suppressed. You can list all projects with 'oc projects' Using project "default". Login successful. You have access to 115 projects, the list has been suppressed. You can list all projects with 'oc projects' Using project "default". Login successful. You have access to 115 projects, the list has been suppressed. You can list all projects with 'oc projects' Using project "default". Login successful. You have access to 115 projects, the list has been suppressed. You can list all projects with 'oc projects' Using project "default". Login successful. You have access to 115 projects, the list has been suppressed. You can list all projects with 'oc projects' Using project "default". Login successful. You have access to 115 projects, the list has been suppressed. You can list all projects with 'oc projects' Using project "default". namespace/openldap created secret/openldap created service/openldap created deployment.apps/openldap created { "kind": "IdentityProvider", "type": "LDAPIdentityProvider", "href": "/api/clusters_mgmt/v1/clusters/25l8qll634ftj2o7o6proa46vke00pml/identity_providers/25l9s7n2b5j0j2jtfjktrjkoc2l1ik8j", "id": "25l9s7n2b5j0j2jtfjktrjkoc2l1ik8j", "name": "ldap-provider-qe", "mapping_method": "claim", "ldap": { "attributes": { "id": [ "dn" ], "name": [ "cn" ], "preferred_username": [ "uid" ] }, "bind_dn": "cn=admin,dc=example,dc=org", "insecure": true, "url": "ldap://openldap.openldap.svc.cluster.local:1389/dc=example,dc=org?uid" } } group.user.openshift.io/rhods-admins created group.user.openshift.io/rhods-users created group.user.openshift.io/rhods-noaccess created Error from server (AlreadyExists): groups.user.openshift.io "dedicated-admins" already exists group.user.openshift.io/rhods-admins added: "ldap-adm1" group.user.openshift.io/rhods-admins added: "ldap-adm2" group.user.openshift.io/rhods-admins added: "ldap-adm3" group.user.openshift.io/rhods-admins added: "ldap-adm4" group.user.openshift.io/rhods-admins added: "ldap-adm5" group.user.openshift.io/rhods-admins added: "ldap-adm6" group.user.openshift.io/rhods-admins added: "ldap-adm7" group.user.openshift.io/rhods-admins added: "ldap-adm8" group.user.openshift.io/rhods-admins added: "ldap-adm9" group.user.openshift.io/rhods-admins added: "ldap-adm10" group.user.openshift.io/rhods-admins added: "ldap-adm11" group.user.openshift.io/rhods-admins added: "ldap-adm12" group.user.openshift.io/rhods-admins added: "ldap-adm13" group.user.openshift.io/rhods-admins added: "ldap-adm14" group.user.openshift.io/rhods-admins added: "ldap-adm15" group.user.openshift.io/rhods-admins added: "ldap-adm16" group.user.openshift.io/rhods-admins added: "ldap-adm17" group.user.openshift.io/rhods-admins added: "ldap-adm18" group.user.openshift.io/rhods-admins added: "ldap-adm19" group.user.openshift.io/rhods-admins added: "ldap-adm20" Added 'dedicated-admins' user 'ldap-adm1' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm2' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm3' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm4' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm5' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm6' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm7' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm8' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm9' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm10' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm11' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm12' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm13' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm14' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm15' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm16' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm17' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm18' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm19' to cluster 'ci-rosa-s-a40b' Added 'dedicated-admins' user 'ldap-adm20' to cluster 'ci-rosa-s-a40b' group.user.openshift.io/rhods-users added: "ldap-usr1" group.user.openshift.io/rhods-users added: "ldap-usr2" group.user.openshift.io/rhods-users added: "ldap-usr3" group.user.openshift.io/rhods-users added: "ldap-usr4" group.user.openshift.io/rhods-users added: "ldap-usr5" group.user.openshift.io/rhods-users added: "ldap-usr6" group.user.openshift.io/rhods-users added: "ldap-usr7" group.user.openshift.io/rhods-users added: "ldap-usr8" group.user.openshift.io/rhods-users added: "ldap-usr9" group.user.openshift.io/rhods-users added: "ldap-usr10" group.user.openshift.io/rhods-users added: "ldap-usr11" group.user.openshift.io/rhods-users added: "ldap-usr12" group.user.openshift.io/rhods-users added: "ldap-usr13" group.user.openshift.io/rhods-users added: "ldap-usr14" group.user.openshift.io/rhods-users added: "ldap-usr15" group.user.openshift.io/rhods-users added: "ldap-usr16" group.user.openshift.io/rhods-users added: "ldap-usr17" group.user.openshift.io/rhods-users added: "ldap-usr18" group.user.openshift.io/rhods-users added: "ldap-usr19" group.user.openshift.io/rhods-users added: "ldap-usr20" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess1" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess2" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess3" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess4" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess5" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess6" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess7" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess8" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess9" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess10" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess11" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess12" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess13" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess14" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess15" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess16" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess17" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess18" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess19" group.user.openshift.io/rhods-noaccess added: "ldap-noaccess20" group.user.openshift.io/rhods-users added: "ldap-special." group.user.openshift.io/rhods-users added: "ldap-special^" group.user.openshift.io/rhods-users added: "ldap-special$" group.user.openshift.io/rhods-users added: "ldap-special*" group.user.openshift.io/rhods-users added: "ldap-special?" group.user.openshift.io/rhods-users added: "ldap-special(" group.user.openshift.io/rhods-users added: "ldap-special)" group.user.openshift.io/rhods-users added: "ldap-special[" group.user.openshift.io/rhods-users added: "ldap-special]" group.user.openshift.io/rhods-users added: "ldap-special{" group.user.openshift.io/rhods-users added: "ldap-special}" group.user.openshift.io/rhods-users added: "ldap-special|" group.user.openshift.io/rhods-users added: "ldap-special@" group.user.openshift.io/rhods-users added: "ldap-special;" sleeping 120sec to wait for IDPs to appear in the OCP login page... -----| ODS-CI is starting the tests run...|----- ods_ci/test-variables.yml INFO: getting RHODS URLs from the cluster as per --set-urls-variables OCP Console URL set to: https://console-openshift-console.apps.ci-rosa-s-a40b.bwgt.s1.devshift.org RHODS Dashboard URL set to: https://rhods-dashboard-redhat-ods-applications.apps.ci-rosa-s-a40b.bwgt.s1.devshift.org RHODS API Server URL set to: https://api.ci-rosa-s-a40b.bwgt.s1.devshift.org:6443 RHODS Prometheus URL set to: https://prometheus-redhat-ods-monitoring.apps.ci-rosa-s-a40b.bwgt.s1.devshift.org INFO: we found a yq executable skipping OC login as per parameter --skip-oclogin ============================================================================== Tests ============================================================================== Tests.Deploy ============================================================================== Tests.Deploy.Installation ============================================================================== Tests.Deploy.Installation.Post Install :: Post install test cases that main... ============================================================================== Verify That CUDA Build Chain Succeeds :: Check Cuda builds are com... | PASS | ------------------------------------------------------------------------------ Verify RHODS Display Name and Version :: Verify consistent rhods d... | PASS | ------------------------------------------------------------------------------ Verify RHODS Notebooks Network Policies :: Verifies that the netwo... | PASS | ------------------------------------------------------------------------------ Tests.Deploy.Installation.Post Install :: Post install test cases ... | PASS | 3 tests, 3 passed, 0 failed ============================================================================== Tests.Deploy.Installation | PASS | 3 tests, 3 passed, 0 failed ============================================================================== Tests.Deploy | PASS | 3 tests, 3 passed, 0 failed ============================================================================== Tests.Monitor And Manage ============================================================================== Tests.Monitor And Manage.Metrics ============================================================================== Tests.Monitor And Manage.Metrics.Metrics :: Test suite testing ODS Metrics ============================================================================== Test Existence of Prometheus Alerting Rules :: Verifies the promet... | PASS | ------------------------------------------------------------------------------ Test Existence of Prometheus Recording Rules :: Verifies the prome... | PASS | ------------------------------------------------------------------------------ Tests.Monitor And Manage.Metrics.Metrics :: Test suite testing ODS... | PASS | 2 tests, 2 passed, 0 failed ============================================================================== Tests.Monitor And Manage.Metrics.Billing Metrics :: Test suite testing ODS ... ============================================================================== Verify OpenShift Monitoring Results Are Correct When Running Undef... | FAIL | Element 'xpath=//*[@data-test-id="perspective-switcher-toggle"]/span/h2' did not appear in 30 seconds. ------------------------------------------------------------------------------ Test Billing Metric (Notebook Cpu Usage) On OpenShift Monitoring :... | FAIL | Element 'xpath=//*[@data-test-id="perspective-switcher-toggle"]/span/h2' did not appear in 30 seconds. ------------------------------------------------------------------------------ Tests.Monitor And Manage.Metrics.Billing Metrics :: Test suite tes... | FAIL | 2 tests, 0 passed, 2 failed ============================================================================== Tests.Monitor And Manage.Metrics | FAIL | 4 tests, 2 passed, 2 failed ============================================================================== Tests.Monitor And Manage.Alerts ============================================================================== Tests.Monitor And Manage.Alerts.Alerts :: RHODS monitoring alerts test suite ============================================================================== Verify All Alerts Severity :: Verifies that all alerts have the ex... | PASS | ------------------------------------------------------------------------------ Verify No Alerts Are Firing Except For DeadManSnitch :: Verifies t... | PASS | ------------------------------------------------------------------------------ Tests.Monitor And Manage.Alerts.Alerts :: RHODS monitoring alerts ... | PASS | 2 tests, 2 passed, 0 failed ============================================================================== Tests.Monitor And Manage.Alerts | PASS | 2 tests, 2 passed, 0 failed ============================================================================== Tests.Monitor And Manage | FAIL | 6 tests, 4 passed, 2 failed ============================================================================== Tests.Ods Dashboard ============================================================================== Tests.Ods Dashboard.Ods Dashboard ============================================================================== Verify RHODS Explore Section Contains Only Expected ISVs :: It ver... | PASS | ------------------------------------------------------------------------------ Verify CSS Style Of Getting Started Descriptions :: Verifies the C... | PASS | ------------------------------------------------------------------------------ Tests.Ods Dashboard.Ods Dashboard | PASS | 2 tests, 2 passed, 0 failed ============================================================================== Tests.Ods Dashboard.Ods Dashboard Projects ============================================================================== Tests.Ods Dashboard.Ods Dashboard Projects.Ods Dashboard Projects :: Suite ... ============================================================================== Verify Data Science Projects Page Is Accessible :: Verifies "Data ... | PASS | ------------------------------------------------------------------------------ Verify User Can Access Jupyter Launcher From DS Project Page :: Ve... | PASS | ------------------------------------------------------------------------------ Verify Workbench Images Have Multiple Versions :: Verifies that wo... | PASS | ------------------------------------------------------------------------------ Tests.Ods Dashboard.Ods Dashboard Projects.Ods Dashboard Projects ... | PASS | 3 tests, 3 passed, 0 failed ============================================================================== Tests.Ods Dashboard.Ods Dashboard Projects.Ods Dashboard Projects Permissio... ============================================================================== Verify User Can Access Permission Tab In Their Owned DS Project ::... | PASS | The Test is executed as part of Suite Setup ------------------------------------------------------------------------------ Tests.Ods Dashboard.Ods Dashboard Projects.Ods Dashboard Projects ... | PASS | 1 test, 1 passed, 0 failed ============================================================================== Tests.Ods Dashboard.Ods Dashboard Projects | PASS | 4 tests, 4 passed, 0 failed ============================================================================== Tests.Ods Dashboard.Model Serving ============================================================================== Tests.Ods Dashboard.Model Serving.Model Serving ============================================================================== Verify Model Serving Installation :: Verifies that the core compon... | PASS | ------------------------------------------------------------------------------ Model not deployed, skipping deletion step during teardown Tests.Ods Dashboard.Model Serving.Model Serving | PASS | 1 test, 1 passed, 0 failed ============================================================================== Tests.Ods Dashboard.Model Serving | PASS | 1 test, 1 passed, 0 failed ============================================================================== Tests.Ods Dashboard | PASS | 7 tests, 7 passed, 0 failed ============================================================================== Tests.Jupyterhub ============================================================================== Tests.Jupyterhub.Base-User-Shutdown-Test :: Test Case that verifies a base ... ============================================================================== Verify Base User Can Stop A Running Server :: Verifies that a base... [ ERROR ] Pods not found in search | PASS | ------------------------------------------------------------------------------ Tests.Jupyterhub.Base-User-Shutdown-Test :: Test Case that verifie... | PASS | 1 test, 1 passed, 0 failed ============================================================================== Tests.Jupyterhub.Multiple-Image-Tags :: Test Suite for the multiple noteboo... ============================================================================== Verify All OOTB Images Have Version Dropdowns :: Verifies all imag... | PASS | ------------------------------------------------------------------------------ Tests.Jupyterhub.Multiple-Image-Tags :: Test Suite for the multipl... | PASS | 1 test, 1 passed, 0 failed ============================================================================== Tests.Jupyterhub.Special-User-Testing ============================================================================== Test Special Usernames | PASS | ------------------------------------------------------------------------------ Tests.Jupyterhub.Special-User-Testing | PASS | 1 test, 1 passed, 0 failed ============================================================================== Tests.Jupyterhub.Test-Minimal-Image ============================================================================== Open RHODS Dashboard | PASS | ------------------------------------------------------------------------------ Can Launch Jupyterhub | PASS | ------------------------------------------------------------------------------ Can Login to Jupyterhub | PASS | ------------------------------------------------------------------------------ Can Spawn Notebook | PASS | ------------------------------------------------------------------------------ Can Launch Python3 Smoke Test Notebook | PASS | ------------------------------------------------------------------------------ Verify Tensorflow Can Be Installed In The Minimal Python Image Via... | PASS | ------------------------------------------------------------------------------ Verify Jupyterlab Server Pods Are Spawned In A Custom Namespace ::... | PASS | ------------------------------------------------------------------------------ Tests.Jupyterhub.Test-Minimal-Image | PASS | 7 tests, 7 passed, 0 failed ============================================================================== Tests.Jupyterhub.Test ============================================================================== Logged Into OpenShift | PASS | ------------------------------------------------------------------------------ Can Launch Jupyterhub | PASS | ------------------------------------------------------------------------------ Can Login To Jupyterhub | PASS | ------------------------------------------------------------------------------ Can Spawn Notebook | PASS | ------------------------------------------------------------------------------ Tests.Jupyterhub.Test | PASS | 4 tests, 4 passed, 0 failed ============================================================================== Tests.Jupyterhub | PASS | 14 tests, 14 passed, 0 failed ============================================================================== Tests.Must Gather ============================================================================== Tests.Must Gather.Test-Must-Gather-Logs ============================================================================== Verify that the must-gather image provides RHODS logs and info | PASS | ------------------------------------------------------------------------------ Tests.Must Gather.Test-Must-Gather-Logs | PASS | 1 test, 1 passed, 0 failed ============================================================================== Tests.Must Gather | PASS | 1 test, 1 passed, 0 failed ============================================================================== Tests.Ai Apps ============================================================================== Tests.Ai Apps.Anaconda Commercial Edition ============================================================================== Tests.Ai Apps.Anaconda Commercial Edition.Anaconda Commercial Edition :: Te... ============================================================================== Verify Anaconda Professional Is Available In RHODS Dashboard Explo... | PASS | ------------------------------------------------------------------------------ Tests.Ai Apps.Anaconda Commercial Edition.Anaconda Commercial Edit... | PASS | 1 test, 1 passed, 0 failed ============================================================================== Tests.Ai Apps.Anaconda Commercial Edition | PASS | 1 test, 1 passed, 0 failed ============================================================================== Tests.Ai Apps.Ibm Watson Studio ============================================================================== Tests.Ai Apps.Ibm Watson Studio.Ibm Watson Studio ============================================================================== Verify IBM Watson Studio Is Available In RHODS Dashboard Explore Page | PASS | ------------------------------------------------------------------------------ Tests.Ai Apps.Ibm Watson Studio.Ibm Watson Studio | PASS | 1 test, 1 passed, 0 failed ============================================================================== Tests.Ai Apps.Ibm Watson Studio | PASS | 1 test, 1 passed, 0 failed ============================================================================== Tests.Ai Apps.Intel Aikit ============================================================================== Tests.Ai Apps.Intel Aikit.Intel Aikit ============================================================================== Verify Intel AIKIT Is Available In RHODS Dashboard Explore Page | PASS | ------------------------------------------------------------------------------ Tests.Ai Apps.Intel Aikit.Intel Aikit | PASS | 1 test, 1 passed, 0 failed ============================================================================== Tests.Ai Apps.Intel Aikit | PASS | 1 test, 1 passed, 0 failed ============================================================================== Tests.Ai Apps.Rhoam ============================================================================== Tests.Ai Apps.Rhoam.Rhoam ============================================================================== Verify RHOAM Is Available In RHODS Dashboard Explore Page | PASS | ------------------------------------------------------------------------------ Tests.Ai Apps.Rhoam.Rhoam | PASS | 1 test, 1 passed, 0 failed ============================================================================== Tests.Ai Apps.Rhoam | PASS | 1 test, 1 passed, 0 failed ============================================================================== Tests.Ai Apps.Openvino ============================================================================== Tests.Ai Apps.Openvino.Openvino ============================================================================== Verify OpenVino Is Available In RHODS Dashboard Explore Page | PASS | ------------------------------------------------------------------------------ Tests.Ai Apps.Openvino.Openvino | PASS | 1 test, 1 passed, 0 failed ============================================================================== Tests.Ai Apps.Openvino | PASS | 1 test, 1 passed, 0 failed ============================================================================== Tests.Ai Apps | PASS | 5 tests, 5 passed, 0 failed ============================================================================== Tests | FAIL | 36 tests, 34 passed, 2 failed ============================================================================== Output: /logs/artifacts/results/ods-ci-2023-08-16-15-35-Ai4dpAZVJ0/output.xml XUnit: /logs/artifacts/results/ods-ci-2023-08-16-15-35-Ai4dpAZVJ0/xunit_test_result.xml Log: /logs/artifacts/results/ods-ci-2023-08-16-15-35-Ai4dpAZVJ0/log.html Report: /logs/artifacts/results/ods-ci-2023-08-16-15-35-Ai4dpAZVJ0/test_report.html 2