application = <iqe.base.application.Application object at 0x7f3ffa2d2870>
product_id = 'rhacs', billing_provider = 'red hat'

    @pytest.mark.ephemeral
    @pytest.mark.ephemeral_only
    @pytest.mark.parametrize(
        "billing_provider, product_id",
        get_swatch_billing_provider_and_payg_product(1, product_type="non-contract"),
    )
    def test_verify_old_month_remittance_with_re_tally_operation(
        application, product_id, billing_provider
    ):
        """Verify re-tally operation for old remittance record.
        metadata:
            assignee: tmcknigh
            negative: false
            importance: high
            bugzilla: SWATCH-2158
            requirements: payg_tally
            test_steps:
                1. Reset the account
                2. Create one mock events for previous months.
                3. Sync hourly tally.
                4. Verify remittance
                5. Create another mock events for the current month
                6. Sync tally again for multiple times
                7. Verify remittance api returns correct usage for all mm products
                8. Verify that the accumulation_period was set from
                the event's timestamp (year and month match)
    
            expected_results:
                1. Remittance value should be correct for all mm products after re-tally operation.
                2. Remittance accumulation_period  should match with event's timestamp.
        """
        application.rhsm_subscriptions.reset_account()
        first_day, last_day = get_monthly_range(iso=False)
        if today().replace(hour=0, minute=0, second=0, microsecond=0).day == last_day.day:
            pytest.skip("Skipping test since timestamps will span two months or a future date")
    
        value = random.uniform(0.5, 20.1)
        metrics = config_parser.get_swatch_metric_id_from_tag_metrics(product_id=product_id)
        metric = random.choice(metrics)
        logging.critical(f"{value} for {metric} for {product_id}")
        start_times_list = [
            today().replace(microsecond=0) - timedelta(days=31),
            today().replace(microsecond=0) - timedelta(days=37),
            today().replace(microsecond=0) + timedelta(days=1, hours=23),
        ]
        start_time = start_times_list[0].strftime("%Y-%m-%dT%H:%M:%SZ")
        end_time = start_times_list[2].strftime("%Y-%m-%dT%H:%M:%SZ")
    
        billing_account_id = "test" + fauxfactory.gen_alpha(3).lower()
        customer_id = str(uuid.uuid4())
        # Create mock subscription for AWS to eventually check billing using moto
        if billing_provider == "aws":
            application.rhsm_subscriptions.create_mock_subscription(
                product_id=product_id,
                billing_provider=billing_provider,
                billing_account_id=billing_account_id,
                customer_id=customer_id,
            )
        billing_factor = config_parser.get_swatch_billing_factor(
            product_id=product_id, metric_id=metric.replace("-", "_")
        )
        data = {
            metric.lower(): value,
            "billing_provider": billing_provider,
            "billing_account_id": billing_account_id,
        }
        # Create one event for previous month
        application.rhsm_subscriptions.create_mock_payg_cluster(
            product_id=product_id, **data, start_time=start_times_list[1]
        )
        log.info(f"Cluster created with {data}")
        application.rhsm_subscriptions.sync_tally_hourly(
            product_id=product_id,
            wait=False,
            perform_metering=False,
            metric_id=metric,
            wait_for_remittance=True,
            start=start_time,
            end=end_time,
        )
    
        prev_month_remittance = application.rhsm_subscriptions.get_account_remittance(
            product_id=product_id,
            metricId=metric,
            billingProvider=billing_provider,
            billingAccountId=billing_account_id,
        )[-1]
        expected_remittance = apply_billing_factor(value, billing_factor)
        # Verify remittance
        assert expected_remittance == int(prev_month_remittance["remittedValue"]), (
            f"Incorrect previous month expected_remittance/actual_remittance  value"
            f"{prev_month_remittance['remittedValue']}/{expected_remittance}"
        )
    
        # Create an event for current month
        application.rhsm_subscriptions.create_mock_payg_cluster(product_id=product_id, **data)
        # re-tally 3 times
        for i in range(0, 2):
            application.rhsm_subscriptions.sync_tally_hourly(
                product_id=product_id,
                wait=False,
                perform_metering=False,
                metric_id=metric,
                wait_time=4,
                timeout_tries=3,
                start=start_time,
                end=end_time,
            )
    
        expected_remittance = expected_remittance * 2
    
>       application.rhsm_subscriptions.wait_for_remittance(
            product_id=product_id,
            metric_id=metric,
            billingProvider=billing_provider,
            billingAccountId=billing_account_id,
            sum_of_pre_remittance=0,
            expected_remittance_value=expected_remittance,
        )

--------------------------------- Captured Log ---------------------------------
2025-04-08 23:06:44 INFO Using random seed value for random generation: 3060
2025-04-08 23:06:44 INFO Flushing usage messages off from billable usage kafka topic
2025-04-08 23:06:44 INFO PUT request response url: http://swatch-billable-usage-service:8000/api/swatch-billable-usage/internal/rpc/topics/flush, status_code: 200
2025-04-08 23:06:44 INFO Trace log using : 00-00000000000000000000000000000000-0000000000000000-00  
2025-04-08 23:06:46 INFO DELETE request response url: http://swatch-tally-service:8000/api/rhsm-subscriptions/v1/internal/rpc/tally/3340851, status_code: 200
2025-04-08 23:06:46 INFO Trace log using : 00-75db7c14d7dd82b04577f411e89e17a3-a9cab2d6d3ca5a11-00  
2025-04-08 23:06:46 INFO Reset moto b'{"status": "ok"}'
2025-04-08 23:06:46 CRITICAL 4.318285281389647 for Cores for rhacs
2025-04-08 23:06:46 INFO Adding payg events of org 3340851 ...
2025-04-08 23:06:46 INFO POST request response url: http://swatch-tally-service:8000/api/rhsm-subscriptions/v1/internal/rpc/tally/events, status_code: 200
2025-04-08 23:06:46 INFO Trace log using : 00-d7870387b7588c337be290c16453a88d-25317efd4161f1d2-00  
2025-04-08 23:06:46 INFO GET request response url: http://swatch-tally-service:8000/api/rhsm-subscriptions/v1/internal/rpc/tally/events/3340851?begin=2025-03-02T23%3A00%3A00Z&end=2025-03-03T00%3A00%3A00Z, status_code: 200
2025-04-08 23:06:46 INFO Trace log using : 00-1b9bbe0b0c70be848de3214f2e15631d-081a6b875e433d77-00  
2025-04-08 23:06:46 INFO Found events after saving: []
2025-04-08 23:06:46 INFO GET request response url: http://swatch-tally-service:8000/api/rhsm-subscriptions/v1/internal/rpc/tally/events/3340851?begin=2025-03-02T23%3A00%3A00Z&end=2025-03-03T00%3A00%3A00Z, status_code: 200
2025-04-08 23:06:46 INFO Trace log using : 00-a9e4f276f361979a0d4fb102c62d2777-4927c2842ba01f51-00  
2025-04-08 23:06:46 INFO Found events after saving: [{'event_source': 'prometheus', 'event_type': 'Cores', 'org_id': '3340851', 'instance_id': '6d89e984-8ff2-4322-94a9-ed293544ac3b', 'event_id': '65010100-352c-405e-8294-213358ff7326', 'service_type': 'Rhacs Cluster', 'timestamp': '2025-03-02T23:00:00Z', 'record_date': '2025-04-08T23:06:46.338715297Z', 'expiration': '2025-03-03T00:00:00Z', 'display_name': 'automation_rhacs_cluster_6d89e984-8ff2-4322-94a9-ed293544ac3b', 'measurements': [{'value': 4.318285281389647, 'metric_id': 'Cores'}], 'role': 'rhacs', 'sla': 'Premium', 'billing_provider': 'red hat', 'billing_account_id': 'testtqw', 'product_tag': ['rhacs'], 'conversion': False}]
2025-04-08 23:06:46 INFO [{'event_source': 'prometheus', 'event_type': 'Cores', 'org_id': '3340851', 'instance_id': '6d89e984-8ff2-4322-94a9-ed293544ac3b', 'timestamp': '2025-03-02T23:00:00Z', 'expiration': '2025-03-03T00:00:00Z', 'display_name': 'automation_rhacs_cluster_6d89e984-8ff2-4322-94a9-ed293544ac3b', 'measurements': [{'value': 4.318285281389647, 'metric_id': 'Cores'}], 'sla': 'Premium', 'service_type': 'Rhacs Cluster', 'role': 'rhacs', 'billing_provider': 'red hat', 'billing_account_id': 'testtqw'}]
2025-04-08 23:06:46 INFO rhacs PAYG Cluster 6d89e984-8ff2-4322-94a9-ed293544ac3b created successfully!!
2025-04-08 23:06:46 INFO Cluster created with {'cores': 4.318285281389647, 'billing_provider': 'red hat', 'billing_account_id': 'testtqw'}
2025-04-08 23:06:46 INFO GET request response url: http://swatch-billable-usage-service:8000/api/swatch-billable-usage/internal/remittance/accountRemittances?productId=rhacs&orgId=3340851&metricId=Cores, status_code: 200
2025-04-08 23:06:46 INFO Trace log using : 00-00000000000000000000000000000000-0000000000000000-00  
2025-04-08 23:06:46 INFO POST request response url: http://swatch-tally-service:8000/api/rhsm-subscriptions/v1/internal/tally/hourly?org=3340851&start=2025-03-08T23%3A06%3A46Z&end=2025-04-10T22%3A06%3A46Z, status_code: 204
2025-04-08 23:06:46 INFO Trace log using : 00-2f910a8df92947e12fab599275b85b2b-9ad4ce71b1a383f7-00  
2025-04-08 23:06:46 INFO Not valid JSON output. Returning plain text message.
2025-04-08 23:06:46 INFO Tally synced from 2025-03-08T23:06:46Z to 2025-04-10T22:06:46Z.
2025-04-08 23:06:46 INFO GET request response url: http://swatch-billable-usage-service:8000/api/swatch-billable-usage/internal/remittance/accountRemittances?productId=rhacs&orgId=3340851&metricId=Cores, status_code: 200
2025-04-08 23:06:46 INFO Trace log using : 00-00000000000000000000000000000000-0000000000000000-00  
2025-04-08 23:06:46 WARNING Action 'wait for remittances for 'rhacs', 'Cores', and '{}'' failed with error 'No new remittance records
assert 0.0 > 0.0'. Retrying...
2025-04-08 23:06:51 WARNING Retrying 'wait for remittances for 'rhacs', 'Cores', and '{}''...
2025-04-08 23:06:51 INFO GET request response url: http://swatch-billable-usage-service:8000/api/swatch-billable-usage/internal/remittance/accountRemittances?productId=rhacs&orgId=3340851&metricId=Cores, status_code: 200
2025-04-08 23:06:51 INFO Trace log using : 00-00000000000000000000000000000000-0000000000000000-00  
2025-04-08 23:06:51 INFO GET request response url: http://swatch-billable-usage-service:8000/api/swatch-billable-usage/internal/remittance/accountRemittances?productId=rhacs&orgId=3340851&metricId=Cores&billingProvider=red+hat&billingAccountId=testtqw, status_code: 200
2025-04-08 23:06:51 INFO Trace log using : 00-00000000000000000000000000000000-0000000000000000-00  
2025-04-08 23:06:51 INFO Adding payg events of org 3340851 ...
2025-04-08 23:06:51 INFO POST request response url: http://swatch-tally-service:8000/api/rhsm-subscriptions/v1/internal/rpc/tally/events, status_code: 200
2025-04-08 23:06:51 INFO Trace log using : 00-b1385dc91bdbe1d79e174db84e2cb37f-ab4f8bd42b34233e-00  
2025-04-08 23:06:51 INFO GET request response url: http://swatch-tally-service:8000/api/rhsm-subscriptions/v1/internal/rpc/tally/events/3340851?begin=2025-04-08T22%3A00%3A00Z&end=2025-04-08T23%3A00%3A00Z, status_code: 200
2025-04-08 23:06:51 INFO Trace log using : 00-9b2b7d727e3b11932e88d13b6d895dc7-ac8c38d16473530b-00  
2025-04-08 23:06:51 INFO Found events after saving: []
2025-04-08 23:06:52 INFO GET request response url: http://swatch-tally-service:8000/api/rhsm-subscriptions/v1/internal/rpc/tally/events/3340851?begin=2025-04-08T22%3A00%3A00Z&end=2025-04-08T23%3A00%3A00Z, status_code: 200
2025-04-08 23:06:52 INFO Trace log using : 00-9146e341eee096651a5be8a37082226a-ac86f5b80b56b97b-00  
2025-04-08 23:06:52 INFO Found events after saving: [{'event_source': 'prometheus', 'event_type': 'Cores', 'org_id': '3340851', 'instance_id': '877a4793-f5fb-41db-b674-09b676014652', 'event_id': '0d012cfd-8bcf-4d98-b2d3-e8661d72ee33', 'service_type': 'Rhacs Cluster', 'timestamp': '2025-04-08T22:00:00Z', 'record_date': '2025-04-08T23:06:51.928440085Z', 'expiration': '2025-04-08T23:00:00Z', 'display_name': 'automation_rhacs_cluster_877a4793-f5fb-41db-b674-09b676014652', 'measurements': [{'value': 4.318285281389647, 'metric_id': 'Cores'}], 'role': 'rhacs', 'sla': 'Premium', 'billing_provider': 'red hat', 'billing_account_id': 'testtqw', 'product_tag': ['rhacs'], 'conversion': False}]
2025-04-08 23:06:52 INFO [{'event_source': 'prometheus', 'event_type': 'Cores', 'org_id': '3340851', 'instance_id': '877a4793-f5fb-41db-b674-09b676014652', 'timestamp': '2025-04-08T22:00:00Z', 'expiration': '2025-04-08T23:00:00Z', 'display_name': 'automation_rhacs_cluster_877a4793-f5fb-41db-b674-09b676014652', 'measurements': [{'value': 4.318285281389647, 'metric_id': 'Cores'}], 'sla': 'Premium', 'service_type': 'Rhacs Cluster', 'role': 'rhacs', 'billing_provider': 'red hat', 'billing_account_id': 'testtqw'}]
2025-04-08 23:06:52 INFO rhacs PAYG Cluster 877a4793-f5fb-41db-b674-09b676014652 created successfully!!
2025-04-08 23:06:52 INFO POST request response url: http://swatch-tally-service:8000/api/rhsm-subscriptions/v1/internal/tally/hourly?org=3340851&start=2025-03-08T23%3A06%3A46Z&end=2025-04-10T22%3A06%3A46Z, status_code: 204
2025-04-08 23:06:52 INFO Trace log using : 00-e44351d4038df984516044ddfd296ee1-5c335b44b4db2870-00  
2025-04-08 23:06:52 INFO Not valid JSON output. Returning plain text message.
2025-04-08 23:06:52 INFO Tally synced from 2025-03-08T23:06:46Z to 2025-04-10T22:06:46Z.
2025-04-08 23:06:52 INFO POST request response url: http://swatch-tally-service:8000/api/rhsm-subscriptions/v1/internal/tally/hourly?org=3340851&start=2025-03-08T23%3A06%3A46Z&end=2025-04-10T22%3A06%3A46Z, status_code: 204
2025-04-08 23:06:52 INFO Trace log using : 00-1e5c0a29c28b02f47e28345321ff20b0-b20f4299dd2f546c-00  
2025-04-08 23:06:52 INFO Not valid JSON output. Returning plain text message.
2025-04-08 23:06:52 INFO Tally synced from 2025-03-08T23:06:46Z to 2025-04-10T22:06:46Z.
2025-04-08 23:06:52 INFO GET request response url: http://swatch-billable-usage-service:8000/api/swatch-billable-usage/internal/remittance/accountRemittances?productId=rhacs&orgId=3340851&metricId=Cores&billingProvider=red+hat&billingAccountId=testtqw, status_code: 200
2025-04-08 23:06:52 INFO Trace log using : 00-00000000000000000000000000000000-0000000000000000-00  
2025-04-08 23:06:52 WARNING Action 'wait for remittances for 'rhacs', 'Cores', and '{'billingProvider': 'red hat', 'billingAccountId': 'testtqw'}'' failed with error 'New remittance records do not match expected sum of 10.0. Found 5.0
assert 5.0 == 10.0'. Retrying...
2025-04-08 23:06:57 WARNING Retrying 'wait for remittances for 'rhacs', 'Cores', and '{'billingProvider': 'red hat', 'billingAccountId': 'testtqw'}''...
2025-04-08 23:06:57 INFO GET request response url: http://swatch-billable-usage-service:8000/api/swatch-billable-usage/internal/remittance/accountRemittances?productId=rhacs&orgId=3340851&metricId=Cores&billingProvider=red+hat&billingAccountId=testtqw, status_code: 200
2025-04-08 23:06:57 INFO Trace log using : 00-00000000000000000000000000000000-0000000000000000-00  
2025-04-08 23:06:57 WARNING Action 'wait for remittances for 'rhacs', 'Cores', and '{'billingProvider': 'red hat', 'billingAccountId': 'testtqw'}'' failed with error 'New remittance records do not match expected sum of 10.0. Found 14.0
assert 14.0 == 10.0'. Retrying...
2025-04-08 23:07:02 WARNING Retrying 'wait for remittances for 'rhacs', 'Cores', and '{'billingProvider': 'red hat', 'billingAccountId': 'testtqw'}''...
2025-04-08 23:07:02 INFO GET request response url: http://swatch-billable-usage-service:8000/api/swatch-billable-usage/internal/remittance/accountRemittances?productId=rhacs&orgId=3340851&metricId=Cores&billingProvider=red+hat&billingAccountId=testtqw, status_code: 200
2025-04-08 23:07:02 INFO Trace log using : 00-00000000000000000000000000000000-0000000000000000-00  
2025-04-08 23:07:02 WARNING Action 'wait for remittances for 'rhacs', 'Cores', and '{'billingProvider': 'red hat', 'billingAccountId': 'testtqw'}'' failed with error 'New remittance records do not match expected sum of 10.0. Found 14.0
assert 14.0 == 10.0'. Retrying...
2025-04-08 23:07:07 WARNING Retrying 'wait for remittances for 'rhacs', 'Cores', and '{'billingProvider': 'red hat', 'billingAccountId': 'testtqw'}''...
2025-04-08 23:07:07 INFO GET request response url: http://swatch-billable-usage-service:8000/api/swatch-billable-usage/internal/remittance/accountRemittances?productId=rhacs&orgId=3340851&metricId=Cores&billingProvider=red+hat&billingAccountId=testtqw, status_code: 200
2025-04-08 23:07:07 INFO Trace log using : 00-00000000000000000000000000000000-0000000000000000-00  
2025-04-08 23:07:07 WARNING Action 'wait for remittances for 'rhacs', 'Cores', and '{'billingProvider': 'red hat', 'billingAccountId': 'testtqw'}'' failed with error 'New remittance records do not match expected sum of 10.0. Found 14.0
assert 14.0 == 10.0'. Retrying...
--------------------------------- Captured Out ---------------------------------