When trying to get value of some metrics, in the UI I can see a value but in the automation I doesn't get value consistently which makes some tests flaky.

4.18

1.
2.
3.

Output from the console, where 'result': [] the metric has no value.

> /home/cloud-user/ohad/cnv-tests/utilities/monitoring.py(173)get_metrics_value()
    171     import pdb
    172     pdb.set_trace()
--> 173     if metric_results:
    174         metric_values_list = [value for metric_val in metric_results for value in metric_val.get("value")]
    175         return metric_values_list[1]

ipdb> prometheus.query(query=metrics_name)
{'status': 'success', 'data': {'resultType': 'vector', 'result': []}}
ipdb> prometheus.query(query=metrics_name)
{'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kubevirt_vmi_migration_data_remaining_bytes', 'container': 'virt-handler', 'endpoint': 'metrics', 'instance': '10.128.2.73:8443', 'job': 'kubevirt-prometheus-metrics', 'name': 'vm-for-migration-metrics-test-1740058126-4457808', 'namespace': 'metrics-test-migration-metrics', 'node': 'iuo-or1-418-gtmsd-worker-0-6wf9m', 'pod': 'virt-handler-89g87', 'service': 'kubevirt-prometheus-metrics'}, 'value': [1740059099.054, '28966912']}]}}
ipdb> prometheus.query(query=metrics_name)
{'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kubevirt_vmi_migration_data_remaining_bytes', 'container': 'virt-handler', 'endpoint': 'metrics', 'instance': '10.128.2.73:8443', 'job': 'kubevirt-prometheus-metrics', 'name': 'vm-for-migration-metrics-test-1740058126-4457808', 'namespace': 'metrics-test-migration-metrics', 'node': 'iuo-or1-418-gtmsd-worker-0-6wf9m', 'pod': 'virt-handler-89g87', 'service': 'kubevirt-prometheus-metrics'}, 'value': [1740059100.105, '28966912']}]}}
ipdb> prometheus.query(query=metrics_name)
{'status': 'success', 'data': {'resultType': 'vector', 'result': []}}
ipdb> prometheus.query(query=metrics_name)
{'status': 'success', 'data': {'resultType': 'vector', 'result': []}}
ipdb> prometheus.query(query=metrics_name)
{'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kubevirt_vmi_migration_data_remaining_bytes', 'container': 'virt-handler', 'endpoint': 'metrics', 'instance': '10.128.2.73:8443', 'job': 'kubevirt-prometheus-metrics', 'name': 'vm-for-migration-metrics-test-1740058126-4457808', 'namespace': 'metrics-test-migration-metrics', 'node': 'iuo-or1-418-gtmsd-worker-0-6wf9m', 'pod': 'virt-handler-89g87', 'service': 'kubevirt-prometheus-metrics'}, 'value': [1740059106.138, '28966912']}]}}
ipdb> prometheus.query(query=metrics_name)
{'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kubevirt_vmi_migration_data_remaining_bytes', 'container': 'virt-handler', 'endpoint': 'metrics', 'instance': '10.128.2.73:8443', 'job': 'kubevirt-prometheus-metrics', 'name': 'vm-for-migration-metrics-test-1740058126-4457808', 'namespace': 'metrics-test-migration-metrics', 'node': 'iuo-or1-418-gtmsd-worker-0-6wf9m', 'pod': 'virt-handler-89g87', 'service': 'kubevirt-prometheus-metrics'}, 'value': [1740059106.948, '28966912']}]}}
ipdb> prometheus.query(query=metrics_name)
{'status': 'success', 'data': {'resultType': 'vector', 'result': []}}
ipdb> prometheus.query(query=metrics_name)
{'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kubevirt_vmi_migration_data_remaining_bytes', 'container': 'virt-handler', 'endpoint': 'metrics', 'instance': '10.128.2.73:8443', 'job': 'kubevirt-prometheus-metrics', 'name': 'vm-for-migration-metrics-test-1740058126-4457808', 'namespace': 'metrics-test-migration-metrics', 'node': 'iuo-or1-418-gtmsd-worker-0-6wf9m', 'pod': 'virt-handler-89g87', 'service': 'kubevirt-prometheus-metrics'}, 'value': [1740059108.8, '28966912']}]}}