Uploaded image for project: 'Subscription Watch'
  1. Subscription Watch
  2. SWATCH-4573

Investigate flaky test "test_verify_rosa_vs_osd_in_swatch"

XMLWordPrintable

    • 5
    • False
    • Hide

      None

      Show
      None
    • False
    • subs-swatch-lightning
    • Swatch Lightning Sprint 10

      This test is randomly failing because the system is failing to report usage data to our simulated AWS billing environment ("Moto") within the expected timeframe, resulting in an empty list of results instead of the recorded usage.

       

      The test "test_verify_rosa_vs_osd_in_swatch" has been randomly failing lately too often.

      application = <iqe.base.application.Application object at 0x7ff638d40950>
      
      
      @pytest.mark.ephemeral
      @pytest.mark.ephemeral_only
      def test_verify_rosa_vs_osd_in_swatch(application):
          """Verify swatch properly differentiates OSD and ROSA
          using internal remittance API.
          metadata:
              assignee: tmcknigh
              negative: false
              importance: high
              requirements: payg_tally
              test_steps:
                  1. Create osd event with some usage
                  2. Create a contract for the current month
                  3. Create a rosa event with same usage,billing_provider and billing_account_id
                  4. Sync hourly tally.
                  5. Verify remittance for both the product
                  6. Verify billing provider filter for both the product
      
              expected_results:
                  1. Remittance should be calculated properly as per products.
                  2. Billing provider should work as expected.
          """
          application.rhsm_subscriptions.reset_account()
          billing_account_id = "test" + fauxfactory.gen_alpha(3).lower()
          sample_data = application.rhsm_subscriptions.create_sample_event_and_contract(
              product_id="rosa",
              tallied_vs_contract_usage="greater",
              billing_account_id=billing_account_id,
          )
          osd_cluster = application.rhsm_subscriptions.create_mock_payg_cluster(
              product_id="OpenShift-dedicated-metrics", **sample_data["usage_data"]
          )
          application.rhsm_subscriptions.sync_tally_hourly(
              product_id="OpenShift-dedicated-metrics",
              perform_metering=False,
              metric_id="Cores",
          )
      
          sleep(5)
          application.rhsm_subscriptions.rhsm_internal_api.billable_usage.flush_billable_usage_topic()
      
          # verify billing_provider filter for both the product
          application.rhsm_subscriptions.wait_for_instance(
              product_id="OpenShift-dedicated-metrics",
              billing_provider="aws",
              display_name_contains=osd_cluster,
          )
          application.rhsm_subscriptions.wait_for_instance(
              product_id="rosa",
              billing_provider="aws",
              display_name_contains=sample_data["cluster"],
          )
          # verify rosa billing usage vs osd billing usage
          # rosa usage should be based on contract
          # osd usage should be based on tally
          remitted_values = []
          for metric in sample_data["metrics"]:
              usage = sample_data["usage_data"][metric.lower()]
              metric_name = metric.lower().replace("-", "_")
              contract = sample_data["contract_data"][metric_name]
              billing_factor = config_parser.get_swatch_billing_factor(
                  product_id="rosa", metric_id=metric.replace("-", "_")
              )
      
              def get_osd_tally_remittances(metric_id=metric):
                  osd_remittance = application.rhsm_subscriptions.get_account_remittance(
                      product_id="OpenShift-dedicated-metrics",
                      metricId=metric,
                      billingProvider="aws",
                      billingAccountId=billing_account_id,
                  )[0]
                  tally = application.rhsm_subscriptions.get_today_tally_report(
                      product_id="OpenShift-dedicated-metrics", metric_id=metric_id
                  )["value"]
                  return [osd_remittance, tally]
      
              def assert_osd_tally_remittances(values):
                  osd_remittance = values[0]
                  tally = values[1]
                  assert osd_remittance["remittedValue"] > 0, "remittance should not zero."
                  assert math.ceil(tally * billing_factor) == int(
                      osd_remittance["remittedValue"] * billing_factor
                  ), f"Incorrect remittance/tally  value - {osd_remittance['remittedValue']}/{tally}"
      
              wait_for(
                  invoke_function=get_osd_tally_remittances,
                  assert_function=assert_osd_tally_remittances,
                  description=f"wait for osd and tally remittances match for metric {metric}",
              )
      
              rosa_remittance = application.rhsm_subscriptions.get_account_remittance(
                  product_id="rosa",
                  metricId=metric,
                  billingProvider="aws",
                  billingAccountId=billing_account_id,
              )[0]
              assert rosa_remittance["remittedValue"] > 0, "remittance should not zero."
              assert (
                  math.ceil((usage - (contract / billing_factor)) * billing_factor)
                  == int(rosa_remittance["remittedValue"]) * billing_factor
              ), (
                  f"Incorrect remittance/contract/usage value - {rosa_remittance['remittedValue']}/"
                  f"{contract}/"
                  f"{usage}"
              )
              remitted_values.append(int(rosa_remittance["remittedValue"]) * billing_factor)
          # Check AWS usage using moto
      
        aws_usages = application.rhsm_subscriptions.verify_aws_usage(billing_provider="aws")
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      
      /iqe_venv/lib/python3.12/site-packages/iqe_rhsm_subscriptions/tests/integration/swatch_billing/test_remitance.py:1006: 
      
      
      
      /iqe_venv/lib/python3.12/site-packages/iqe_rhsm_subscriptions/init.py:3330: in verify_aws_usage
          aws_usages = self.get_moto_aws_billing(expected_usages)
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      /iqe_venv/lib/python3.12/site-packages/iqe_rhsm_subscriptions/init.py:3307: in get_moto_aws_billing
          aws_usage = wait_for(
      
      
      
      invoke_function = <function ApplicationRhsmSubscriptions.get_moto_aws_billing..get_aws_usage at 0x7ff6304d23e0>
      assert_function = <function ApplicationRhsmSubscriptions.get_moto_aws_billing..assert_aws_usage at 0x7ff6304d0d60>
      description = 'wait for aws usages', sleep_time = 6, max_retries = 5
      failure_function = None
      
      
      def wait_for(
          invoke_function,
          assert_function,
          description,
          sleep_time=2,
          max_retries=3,
          failure_function=None,
      ):
          """
          Repeatedly invokes a function and checks its output using an assertion function.
      
          Args:
              invoke_function (callable): The function to be invoked repeatedly.
              assert_function (callable): A function that takes the output of invoke_function
                                          and returns True if the condition is met, otherwise False.
              failure_function (callable): A function that is invoked when the action is timeout.
              timeout (int): Maximum time to wait for the assertion to pass, in seconds. Default is 10.
              interval (float): Time to wait between invocations, in seconds. Default is 0.5.
      
          Returns:
              The output of invoke_function if assert_function returns True within the timeout.
          """
          retries = 0
          result = None
          while retries <= max_retries:
              if retries > 0:
                  logging.warning(f"Retrying '{description}'...")
              try:
                  result = invoke_function()
                  assert_function(result)
                  logging.info(f"The action '{description}' succeeded with: {result}")
                  return result
              except Exception as e:
                  logging.warning(f"Action '{description}' failed with error '{e}'. Retrying...")
                  retries += 1
                  sleep(sleep_time)
      
          error_message = f"The action '{description}' was not completed within the timeout period. Last result was '{result}'. "
          logging.error(error_message)
          if failure_function:
              failure_function(result)
      
        assert False, error_message
      E       AssertionError: The action 'wait for aws usages' was not completed within the timeout period. Last result was '[]'.
      E       assert False
      
      
      
      /iqe_venv/lib/python3.12/site-packages/iqe_rhsm_subscriptions/utils/test_framework_utils.py:51: AssertionError
      

      Link: https://reportportal-smqe.apps.dno.ocp-hub.prod.psi.redhat.com/ui/#subscription_watch/launches/1934/399191/32781021/32781233/log?item0Params=filter.eq.hasStats%3Dtrue%26filter.eq.hasChildren%3Dfalse%26filter.in.type%3DSTEP%26filter.in.status%3DFAILED%252CINTERRUPTED&logParams=filter.gte.level%3DTRACE%26page.page%3D1

      Acceptance Criteria

      • Investigate and fix the IQE failure

              jcarvaja@redhat.com Jose Carvajal Hilario
              jcarvaja@redhat.com Jose Carvajal Hilario
              Votes:
              0 Vote for this issue
              Watchers:
              3 Start watching this issue

                Created:
                Updated:
                Resolved: