repomirrorworker stdout | 2022-06-03 12:42:01,435 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:42:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:42:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:42:33.733973+00:00 (in 29.999173 seconds)
repomirrorworker stdout | 2022-06-03 12:42:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:42:33 UTC)" (scheduled at 2022-06-03 12:42:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:42:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:42:03,751 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:42:03,755 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 42, 3, 753812), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 42, 3, 753812), 0, 2, datetime.datetime(2022, 6, 3, 12, 42, 3, 753812), True, 8, 12])
repomirrorworker stdout | 2022-06-03 12:42:03,758 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 12:42:03,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 42, 3, 759100), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 42, 3, 759100), 0, 2, datetime.datetime(2022, 6, 3, 12, 42, 3, 759100), True, 3, 7])
repomirrorworker stdout | 2022-06-03 12:42:03,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 3-7
repomirrorworker stdout | 2022-06-03 12:42:03,764 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 42, 3, 763150), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 42, 3, 763150), 0, 2, datetime.datetime(2022, 6, 3, 12, 42, 3, 763150), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:42:03,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:42:03,767 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 42, 3, 766549), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 42, 3, 766549), 0, 2, datetime.datetime(2022, 6, 3, 12, 42, 3, 766549), True, 12, 15])
repomirrorworker stdout | 2022-06-03 12:42:03,770 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 12-15
repomirrorworker stdout | 2022-06-03 12:42:03,771 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 42, 3, 770480), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 42, 3, 770480), 0, 2, datetime.datetime(2022, 6, 3, 12, 42, 3, 770480), True, 7, 11])
repomirrorworker stdout | 2022-06-03 12:42:03,773 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 12:42:03,773 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:42:03,773 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:42:03,774 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:42:03,783 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:42:03,783 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:42:03,783 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:42:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:42:31,496 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:42:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:42:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:43:03.733973+00:00 (in 29.999073 seconds)
repomirrorworker stdout | 2022-06-03 12:42:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:43:03 UTC)" (scheduled at 2022-06-03 12:42:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:42:33,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:42:33,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:42:33,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 42, 33, 761119), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 42, 33, 761119), 0, 2, datetime.datetime(2022, 6, 3, 12, 42, 33, 761119), True, 9, 13])
repomirrorworker stdout | 2022-06-03 12:42:33,767 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 12:42:33,768 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 42, 33, 767614), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 42, 33, 767614), 0, 2, datetime.datetime(2022, 6, 3, 12, 42, 33, 767614), True, 13, 15])
repomirrorworker stdout | 2022-06-03 12:42:33,771 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 13-15
repomirrorworker stdout | 2022-06-03 12:42:33,772 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 42, 33, 771603), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 42, 33, 771603), 0, 2, datetime.datetime(2022, 6, 3, 12, 42, 33, 771603), True, 5, 9])
repomirrorworker stdout | 2022-06-03 12:42:33,775 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 12:42:33,776 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 42, 33, 775411), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 42, 33, 775411), 0, 2, datetime.datetime(2022, 6, 3, 12, 42, 33, 775411), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:42:33,779 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:42:33,779 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:42:33,779 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:42:33,779 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:42:33,789 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:42:33,789 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:42:33,789 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:43:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:43:01,554 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:43:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:43:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:43:33.733973+00:00 (in 29.999176 seconds)
repomirrorworker stdout | 2022-06-03 12:43:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:43:33 UTC)" (scheduled at 2022-06-03 12:43:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:43:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:43:03,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:43:03,751 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 43, 3, 750000), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 43, 3, 750000), 0, 2, datetime.datetime(2022, 6, 3, 12, 43, 3, 750000), True, 6, 10])
repomirrorworker stdout | 2022-06-03 12:43:03,755 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 12:43:03,756 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 43, 3, 755327), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 43, 3, 755327), 0, 2, datetime.datetime(2022, 6, 3, 12, 43, 3, 755327), True, 2, 6])
repomirrorworker stdout | 2022-06-03 12:43:03,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 12:43:03,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 43, 3, 759331), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 43, 3, 759331), 0, 2, datetime.datetime(2022, 6, 3, 12, 43, 3, 759331), True, 10, 14])
repomirrorworker stdout | 2022-06-03 12:43:03,763 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 10-14
repomirrorworker stdout | 2022-06-03 12:43:03,764 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 43, 3, 763312), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 43, 3, 763312), 0, 2, datetime.datetime(2022, 6, 3, 12, 43, 3, 763312), True, 14, 15])
repomirrorworker stdout | 2022-06-03 12:43:03,767 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 14-15
repomirrorworker stdout | 2022-06-03 12:43:03,768 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 43, 3, 767171), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 43, 3, 767171), 0, 2, datetime.datetime(2022, 6, 3, 12, 43, 3, 767171), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:43:03,770 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:43:03,771 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:43:03,771 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:43:03,771 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:43:03,781 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:43:03,781 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:43:03,781 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:43:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:43:31,609 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:43:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:43:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:44:03.733973+00:00 (in 29.998899 seconds)
repomirrorworker stdout | 2022-06-03 12:43:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:44:03 UTC)" (scheduled at 2022-06-03 12:43:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:43:33,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:43:33,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:43:33,752 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 43, 33, 751177), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 43, 33, 751177), 0, 2, datetime.datetime(2022, 6, 3, 12, 43, 33, 751177), True, 8, 12])
repomirrorworker stdout | 2022-06-03 12:43:33,756 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 12:43:33,757 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 43, 33, 756177), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 43, 33, 756177), 0, 2, datetime.datetime(2022, 6, 3, 12, 43, 33, 756177), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:43:33,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:43:33,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 43, 33, 759716), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 43, 33, 759716), 0, 2, datetime.datetime(2022, 6, 3, 12, 43, 33, 759716), True, 12, 15])
repomirrorworker stdout | 2022-06-03 12:43:33,763 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 12-15
repomirrorworker stdout | 2022-06-03 12:43:33,764 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 43, 33, 763396), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 43, 33, 763396), 0, 2, datetime.datetime(2022, 6, 3, 12, 43, 33, 763396), True, 5, 9])
repomirrorworker stdout | 2022-06-03 12:43:33,767 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 12:43:33,767 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:43:33,767 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:43:33,768 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:43:33,778 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:43:33,779 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:43:33,779 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:44:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:44:01,677 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:44:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:44:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:44:33.733973+00:00 (in 29.999063 seconds)
repomirrorworker stdout | 2022-06-03 12:44:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:44:33 UTC)" (scheduled at 2022-06-03 12:44:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:44:03,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:44:03,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:44:03,752 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 44, 3, 751631), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 44, 3, 751631), 0, 2, datetime.datetime(2022, 6, 3, 12, 44, 3, 751631), True, 8, 12])
repomirrorworker stdout | 2022-06-03 12:44:03,756 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 12:44:03,757 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 44, 3, 756885), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 44, 3, 756885), 0, 2, datetime.datetime(2022, 6, 3, 12, 44, 3, 756885), True, 12, 15])
repomirrorworker stdout | 2022-06-03 12:44:03,760 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 12-15
repomirrorworker stdout | 2022-06-03 12:44:03,761 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 44, 3, 760355), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 44, 3, 760355), 0, 2, datetime.datetime(2022, 6, 3, 12, 44, 3, 760355), True, 3, 7])
repomirrorworker stdout | 2022-06-03 12:44:03,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 3-7
repomirrorworker stdout | 2022-06-03 12:44:03,765 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 44, 3, 764503), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 44, 3, 764503), 0, 2, datetime.datetime(2022, 6, 3, 12, 44, 3, 764503), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:44:03,768 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:44:03,769 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 44, 3, 768248), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 44, 3, 768248), 0, 2, datetime.datetime(2022, 6, 3, 12, 44, 3, 768248), True, 7, 11])
repomirrorworker stdout | 2022-06-03 12:44:03,772 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 12:44:03,772 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:44:03,772 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:44:03,772 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:44:03,781 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:44:03,781 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:44:03,781 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:44:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:44:31,714 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:44:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:44:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:45:03.733973+00:00 (in 29.999099 seconds)
repomirrorworker stdout | 2022-06-03 12:44:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:45:03 UTC)" (scheduled at 2022-06-03 12:44:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:44:33,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:44:33,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:44:33,753 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 44, 33, 751973), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 44, 33, 751973), 0, 2, datetime.datetime(2022, 6, 3, 12, 44, 33, 751973), True, 11, 15])
repomirrorworker stdout | 2022-06-03 12:44:33,757 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 12:44:33,759 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 44, 33, 758021), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 44, 33, 758021), 0, 2, datetime.datetime(2022, 6, 3, 12, 44, 33, 758021), True, 7, 11])
repomirrorworker stdout | 2022-06-03 12:44:33,761 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 12:44:33,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 44, 33, 761985), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 44, 33, 761985), 0, 2, datetime.datetime(2022, 6, 3, 12, 44, 33, 761985), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:44:33,765 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:44:33,766 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 44, 33, 765673), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 44, 33, 765673), 0, 2, datetime.datetime(2022, 6, 3, 12, 44, 33, 765673), True, 5, 9])
repomirrorworker stdout | 2022-06-03 12:44:33,769 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 12:44:33,769 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:44:33,769 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:44:33,769 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:44:33,780 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:44:33,780 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:44:33,780 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:45:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:45:01,772 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:45:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:45:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:45:33.733973+00:00 (in 29.998945 seconds)
repomirrorworker stdout | 2022-06-03 12:45:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:45:33 UTC)" (scheduled at 2022-06-03 12:45:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:45:03,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:45:03,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:45:03,753 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 45, 3, 751961), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 45, 3, 751961), 0, 2, datetime.datetime(2022, 6, 3, 12, 45, 3, 751961), True, 8, 12])
repomirrorworker stdout | 2022-06-03 12:45:03,756 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 12:45:03,758 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 45, 3, 757114), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 45, 3, 757114), 0, 2, datetime.datetime(2022, 6, 3, 12, 45, 3, 757114), True, 12, 15])
repomirrorworker stdout | 2022-06-03 12:45:03,760 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 12-15
repomirrorworker stdout | 2022-06-03 12:45:03,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 45, 3, 761214), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 45, 3, 761214), 0, 2, datetime.datetime(2022, 6, 3, 12, 45, 3, 761214), True, 3, 7])
repomirrorworker stdout | 2022-06-03 12:45:03,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 3-7
repomirrorworker stdout | 2022-06-03 12:45:03,765 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 45, 3, 764772), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 45, 3, 764772), 0, 2, datetime.datetime(2022, 6, 3, 12, 45, 3, 764772), True, 7, 11])
repomirrorworker stdout | 2022-06-03 12:45:03,769 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 12:45:03,770 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 45, 3, 769676), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 45, 3, 769676), 0, 2, datetime.datetime(2022, 6, 3, 12, 45, 3, 769676), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:45:03,773 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:45:03,773 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:45:03,773 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:45:03,773 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:45:03,784 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:45:03,784 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:45:03,784 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:45:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:45:31,833 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:45:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:45:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:46:03.733973+00:00 (in 29.998791 seconds)
repomirrorworker stdout | 2022-06-03 12:45:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:46:03 UTC)" (scheduled at 2022-06-03 12:45:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:45:33,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:45:33,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:45:33,753 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 45, 33, 752039), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 45, 33, 752039), 0, 2, datetime.datetime(2022, 6, 3, 12, 45, 33, 752039), True, 10, 14])
repomirrorworker stdout | 2022-06-03 12:45:33,756 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 10-14
repomirrorworker stdout | 2022-06-03 12:45:33,758 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 45, 33, 757062), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 45, 33, 757062), 0, 2, datetime.datetime(2022, 6, 3, 12, 45, 33, 757062), True, 5, 9])
repomirrorworker stdout | 2022-06-03 12:45:33,760 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 12:45:33,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 45, 33, 760866), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 45, 33, 760866), 0, 2, datetime.datetime(2022, 6, 3, 12, 45, 33, 760866), True, 9, 13])
repomirrorworker stdout | 2022-06-03 12:45:33,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 12:45:33,766 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 45, 33, 765053), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 45, 33, 765053), 0, 2, datetime.datetime(2022, 6, 3, 12, 45, 33, 765053), True, 14, 15])
repomirrorworker stdout | 2022-06-03 12:45:33,768 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 14-15
repomirrorworker stdout | 2022-06-03 12:45:33,770 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 45, 33, 768985), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 45, 33, 768985), 0, 2, datetime.datetime(2022, 6, 3, 12, 45, 33, 768985), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:45:33,772 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:45:33,773 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:45:33,773 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:45:33,773 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:45:33,799 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:45:33,799 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:45:33,799 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:46:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:46:01,876 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:46:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:46:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:46:33.733973+00:00 (in 29.999126 seconds)
repomirrorworker stdout | 2022-06-03 12:46:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:46:33 UTC)" (scheduled at 2022-06-03 12:46:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:46:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:46:03,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:46:03,751 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 46, 3, 749929), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 46, 3, 749929), 0, 2, datetime.datetime(2022, 6, 3, 12, 46, 3, 749929), True, 7, 11])
repomirrorworker stdout | 2022-06-03 12:46:03,754 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 12:46:03,756 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 46, 3, 755041), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 46, 3, 755041), 0, 2, datetime.datetime(2022, 6, 3, 12, 46, 3, 755041), True, 11, 15])
repomirrorworker stdout | 2022-06-03 12:46:03,758 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 12:46:03,759 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 46, 3, 758531), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 46, 3, 758531), 0, 2, datetime.datetime(2022, 6, 3, 12, 46, 3, 758531), True, 2, 6])
repomirrorworker stdout | 2022-06-03 12:46:03,761 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 12:46:03,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 46, 3, 761788), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 46, 3, 761788), 0, 2, datetime.datetime(2022, 6, 3, 12, 46, 3, 761788), True, 6, 10])
repomirrorworker stdout | 2022-06-03 12:46:03,765 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 12:46:03,766 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 46, 3, 765319), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 46, 3, 765319), 0, 2, datetime.datetime(2022, 6, 3, 12, 46, 3, 765319), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:46:03,768 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:46:03,768 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:46:03,768 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:46:03,769 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:46:03,778 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:46:03,778 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:46:03,778 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:46:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:46:31,937 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:46:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:46:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:47:03.733973+00:00 (in 29.999219 seconds)
repomirrorworker stdout | 2022-06-03 12:46:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:47:03 UTC)" (scheduled at 2022-06-03 12:46:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:46:33,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:46:33,747 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:46:33,750 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 46, 33, 749637), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 46, 33, 749637), 0, 2, datetime.datetime(2022, 6, 3, 12, 46, 33, 749637), True, 6, 10])
repomirrorworker stdout | 2022-06-03 12:46:33,754 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 12:46:33,755 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 46, 33, 754795), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 46, 33, 754795), 0, 2, datetime.datetime(2022, 6, 3, 12, 46, 33, 754795), True, 10, 14])
repomirrorworker stdout | 2022-06-03 12:46:33,758 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 10-14
repomirrorworker stdout | 2022-06-03 12:46:33,759 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 46, 33, 758262), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 46, 33, 758262), 0, 2, datetime.datetime(2022, 6, 3, 12, 46, 33, 758262), True, 14, 15])
repomirrorworker stdout | 2022-06-03 12:46:33,761 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 14-15
repomirrorworker stdout | 2022-06-03 12:46:33,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 46, 33, 761482), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 46, 33, 761482), 0, 2, datetime.datetime(2022, 6, 3, 12, 46, 33, 761482), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:46:33,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:46:33,765 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 46, 33, 764653), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 46, 33, 764653), 0, 2, datetime.datetime(2022, 6, 3, 12, 46, 33, 764653), True, 5, 9])
repomirrorworker stdout | 2022-06-03 12:46:33,783 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 12:46:33,783 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:46:33,783 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:46:33,784 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:46:33,794 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:46:33,794 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:46:33,794 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:47:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:47:01,991 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:47:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:47:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:47:33.733973+00:00 (in 29.999221 seconds)
repomirrorworker stdout | 2022-06-03 12:47:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:47:33 UTC)" (scheduled at 2022-06-03 12:47:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:47:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:47:03,747 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:47:03,750 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 47, 3, 749739), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 47, 3, 749739), 0, 2, datetime.datetime(2022, 6, 3, 12, 47, 3, 749739), True, 3, 7])
repomirrorworker stdout | 2022-06-03 12:47:03,754 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 3-7
repomirrorworker stdout | 2022-06-03 12:47:03,755 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 47, 3, 754697), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 47, 3, 754697), 0, 2, datetime.datetime(2022, 6, 3, 12, 47, 3, 754697), True, 11, 15])
repomirrorworker stdout | 2022-06-03 12:47:03,758 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 12:47:03,759 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 47, 3, 758208), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 47, 3, 758208), 0, 2, datetime.datetime(2022, 6, 3, 12, 47, 3, 758208), True, 7, 11])
repomirrorworker stdout | 2022-06-03 12:47:03,763 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 12:47:03,764 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 47, 3, 763920), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 47, 3, 763920), 0, 2, datetime.datetime(2022, 6, 3, 12, 47, 3, 763920), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:47:03,767 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:47:03,767 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:47:03,767 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:47:03,767 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:47:03,776 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:47:03,776 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:47:03,776 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:47:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:47:32,035 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:47:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:47:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:48:03.733973+00:00 (in 29.999219 seconds)
repomirrorworker stdout | 2022-06-03 12:47:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:48:03 UTC)" (scheduled at 2022-06-03 12:47:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:47:33,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:47:33,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:47:33,752 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 47, 33, 751013), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 47, 33, 751013), 0, 2, datetime.datetime(2022, 6, 3, 12, 47, 33, 751013), True, 2, 6])
repomirrorworker stdout | 2022-06-03 12:47:33,755 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 12:47:33,757 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 47, 33, 756063), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 47, 33, 756063), 0, 2, datetime.datetime(2022, 6, 3, 12, 47, 33, 756063), True, 7, 11])
repomirrorworker stdout | 2022-06-03 12:47:33,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 12:47:33,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 47, 33, 759519), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 47, 33, 759519), 0, 2, datetime.datetime(2022, 6, 3, 12, 47, 33, 759519), True, 6, 10])
repomirrorworker stdout | 2022-06-03 12:47:33,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 12:47:33,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 47, 33, 762949), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 47, 33, 762949), 0, 2, datetime.datetime(2022, 6, 3, 12, 47, 33, 762949), True, 11, 15])
repomirrorworker stdout | 2022-06-03 12:47:33,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 12:47:33,767 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 47, 33, 766349), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 47, 33, 766349), 0, 2, datetime.datetime(2022, 6, 3, 12, 47, 33, 766349), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:47:33,769 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:47:33,769 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:47:33,769 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:47:33,769 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:47:33,779 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:47:33,779 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:47:33,780 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:48:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:48:02,083 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:48:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:48:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:48:33.733973+00:00 (in 29.999178 seconds)
repomirrorworker stdout | 2022-06-03 12:48:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:48:33 UTC)" (scheduled at 2022-06-03 12:48:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:48:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:48:03,765 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:48:03,768 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 48, 3, 767481), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 48, 3, 767481), 0, 2, datetime.datetime(2022, 6, 3, 12, 48, 3, 767481), True, 3, 7])
repomirrorworker stdout | 2022-06-03 12:48:03,772 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 3-7
repomirrorworker stdout | 2022-06-03 12:48:03,774 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 48, 3, 772946), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 48, 3, 772946), 0, 2, datetime.datetime(2022, 6, 3, 12, 48, 3, 772946), True, 8, 12])
repomirrorworker stdout | 2022-06-03 12:48:03,776 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 12:48:03,777 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 48, 3, 776507), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 48, 3, 776507), 0, 2, datetime.datetime(2022, 6, 3, 12, 48, 3, 776507), True, 7, 11])
repomirrorworker stdout | 2022-06-03 12:48:03,779 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 12:48:03,780 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 48, 3, 779636), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 48, 3, 779636), 0, 2, datetime.datetime(2022, 6, 3, 12, 48, 3, 779636), True, 12, 15])
repomirrorworker stdout | 2022-06-03 12:48:03,782 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 12-15
repomirrorworker stdout | 2022-06-03 12:48:03,783 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 48, 3, 782880), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 48, 3, 782880), 0, 2, datetime.datetime(2022, 6, 3, 12, 48, 3, 782880), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:48:03,786 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:48:03,786 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:48:03,786 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:48:03,786 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:48:03,796 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:48:03,796 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:48:03,796 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:48:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:48:32,145 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:48:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:48:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:49:03.733973+00:00 (in 29.998980 seconds)
repomirrorworker stdout | 2022-06-03 12:48:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:49:03 UTC)" (scheduled at 2022-06-03 12:48:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:48:33,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:48:33,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:48:33,751 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 48, 33, 750430), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 48, 33, 750430), 0, 2, datetime.datetime(2022, 6, 3, 12, 48, 33, 750430), True, 7, 11])
repomirrorworker stdout | 2022-06-03 12:48:33,755 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 12:48:33,756 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 48, 33, 755885), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 48, 33, 755885), 0, 2, datetime.datetime(2022, 6, 3, 12, 48, 33, 755885), True, 2, 6])
repomirrorworker stdout | 2022-06-03 12:48:33,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 12:48:33,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 48, 33, 759356), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 48, 33, 759356), 0, 2, datetime.datetime(2022, 6, 3, 12, 48, 33, 759356), True, 6, 10])
repomirrorworker stdout | 2022-06-03 12:48:33,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 12:48:33,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 48, 33, 762936), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 48, 33, 762936), 0, 2, datetime.datetime(2022, 6, 3, 12, 48, 33, 762936), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:48:33,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:48:33,767 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 48, 33, 766156), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 48, 33, 766156), 0, 2, datetime.datetime(2022, 6, 3, 12, 48, 33, 766156), True, 11, 15])
repomirrorworker stdout | 2022-06-03 12:48:33,770 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 12:48:33,770 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:48:33,770 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:48:33,770 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:48:33,779 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:48:33,779 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:48:33,779 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:49:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:49:02,200 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:49:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:49:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:49:33.733973+00:00 (in 29.999158 seconds)
repomirrorworker stdout | 2022-06-03 12:49:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:49:33 UTC)" (scheduled at 2022-06-03 12:49:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:49:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:49:03,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:49:03,753 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 49, 3, 752116), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 49, 3, 752116), 0, 2, datetime.datetime(2022, 6, 3, 12, 49, 3, 752116), True, 5, 9])
repomirrorworker stdout | 2022-06-03 12:49:03,757 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 12:49:03,758 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 49, 3, 757633), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 49, 3, 757633), 0, 2, datetime.datetime(2022, 6, 3, 12, 49, 3, 757633), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:49:03,761 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:49:03,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 49, 3, 761599), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 49, 3, 761599), 0, 2, datetime.datetime(2022, 6, 3, 12, 49, 3, 761599), True, 11, 15])
repomirrorworker stdout | 2022-06-03 12:49:03,765 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 12:49:03,766 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 49, 3, 765723), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 49, 3, 765723), 0, 2, datetime.datetime(2022, 6, 3, 12, 49, 3, 765723), True, 9, 13])
repomirrorworker stdout | 2022-06-03 12:49:03,769 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 12:49:03,769 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:49:03,769 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:49:03,769 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:49:03,778 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:49:03,778 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:49:03,778 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:49:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:49:32,241 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:49:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:49:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:50:03.733973+00:00 (in 29.999080 seconds)
repomirrorworker stdout | 2022-06-03 12:49:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:50:03 UTC)" (scheduled at 2022-06-03 12:49:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:49:33,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:49:33,747 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:49:33,750 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 49, 33, 749549), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 49, 33, 749549), 0, 2, datetime.datetime(2022, 6, 3, 12, 49, 33, 749549), True, 5, 9])
repomirrorworker stdout | 2022-06-03 12:49:33,754 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 12:49:33,755 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 49, 33, 754464), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 49, 33, 754464), 0, 2, datetime.datetime(2022, 6, 3, 12, 49, 33, 754464), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:49:33,758 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:49:33,759 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 49, 33, 758205), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 49, 33, 758205), 0, 2, datetime.datetime(2022, 6, 3, 12, 49, 33, 758205), True, 9, 13])
repomirrorworker stdout | 2022-06-03 12:49:33,761 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 12:49:33,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 49, 33, 761456), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 49, 33, 761456), 0, 2, datetime.datetime(2022, 6, 3, 12, 49, 33, 761456), True, 13, 15])
repomirrorworker stdout | 2022-06-03 12:49:33,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 13-15
repomirrorworker stdout | 2022-06-03 12:49:33,764 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:49:33,765 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:49:33,765 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:49:33,774 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:49:33,774 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:49:33,775 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:50:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:50:02,307 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:50:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:50:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:50:33.733973+00:00 (in 29.998996 seconds)
repomirrorworker stdout | 2022-06-03 12:50:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:50:33 UTC)" (scheduled at 2022-06-03 12:50:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:50:03,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:50:03,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:50:03,752 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 50, 3, 751377), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 50, 3, 751377), 0, 2, datetime.datetime(2022, 6, 3, 12, 50, 3, 751377), True, 7, 11])
repomirrorworker stdout | 2022-06-03 12:50:03,756 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 12:50:03,758 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 50, 3, 756821), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 50, 3, 756821), 0, 2, datetime.datetime(2022, 6, 3, 12, 50, 3, 756821), True, 11, 15])
repomirrorworker stdout | 2022-06-03 12:50:03,760 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 12:50:03,761 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 50, 3, 760619), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 50, 3, 760619), 0, 2, datetime.datetime(2022, 6, 3, 12, 50, 3, 760619), True, 2, 6])
repomirrorworker stdout | 2022-06-03 12:50:03,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 12:50:03,765 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 50, 3, 764306), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 50, 3, 764306), 0, 2, datetime.datetime(2022, 6, 3, 12, 50, 3, 764306), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:50:03,768 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:50:03,769 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 50, 3, 768541), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 50, 3, 768541), 0, 2, datetime.datetime(2022, 6, 3, 12, 50, 3, 768541), True, 6, 10])
repomirrorworker stdout | 2022-06-03 12:50:03,771 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 12:50:03,772 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:50:03,772 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:50:03,772 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:50:03,781 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:50:03,781 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:50:03,782 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:50:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:50:32,363 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:50:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:50:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:51:03.733973+00:00 (in 29.999117 seconds)
repomirrorworker stdout | 2022-06-03 12:50:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:51:03 UTC)" (scheduled at 2022-06-03 12:50:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:50:33,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:50:33,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:50:33,754 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 50, 33, 752412), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 50, 33, 752412), 0, 2, datetime.datetime(2022, 6, 3, 12, 50, 33, 752412), True, 5, 9])
repomirrorworker stdout | 2022-06-03 12:50:33,758 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 12:50:33,759 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 50, 33, 758637), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 50, 33, 758637), 0, 2, datetime.datetime(2022, 6, 3, 12, 50, 33, 758637), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:50:33,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:50:33,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 50, 33, 762320), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 50, 33, 762320), 0, 2, datetime.datetime(2022, 6, 3, 12, 50, 33, 762320), True, 10, 14])
repomirrorworker stdout | 2022-06-03 12:50:33,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 10-14
repomirrorworker stdout | 2022-06-03 12:50:33,768 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 50, 33, 767190), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 50, 33, 767190), 0, 2, datetime.datetime(2022, 6, 3, 12, 50, 33, 767190), True, 14, 15])
repomirrorworker stdout | 2022-06-03 12:50:33,770 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 14-15
repomirrorworker stdout | 2022-06-03 12:50:33,771 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 50, 33, 770935), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 50, 33, 770935), 0, 2, datetime.datetime(2022, 6, 3, 12, 50, 33, 770935), True, 9, 13])
repomirrorworker stdout | 2022-06-03 12:50:33,774 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 12:50:33,774 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:50:33,774 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:50:33,775 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:50:33,785 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:50:33,785 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:50:33,785 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:51:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:51:02,421 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:51:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:51:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:51:33.733973+00:00 (in 29.999043 seconds)
repomirrorworker stdout | 2022-06-03 12:51:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:51:33 UTC)" (scheduled at 2022-06-03 12:51:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:51:03,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:51:03,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:51:03,752 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 51, 3, 751245), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 51, 3, 751245), 0, 2, datetime.datetime(2022, 6, 3, 12, 51, 3, 751245), True, 9, 13])
repomirrorworker stdout | 2022-06-03 12:51:03,756 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 12:51:03,757 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 51, 3, 756713), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 51, 3, 756713), 0, 2, datetime.datetime(2022, 6, 3, 12, 51, 3, 756713), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:51:03,761 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:51:03,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 51, 3, 761789), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 51, 3, 761789), 0, 2, datetime.datetime(2022, 6, 3, 12, 51, 3, 761789), True, 5, 9])
repomirrorworker stdout | 2022-06-03 12:51:03,765 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 12:51:03,766 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 51, 3, 765733), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 51, 3, 765733), 0, 2, datetime.datetime(2022, 6, 3, 12, 51, 3, 765733), True, 13, 15])
repomirrorworker stdout | 2022-06-03 12:51:03,769 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 13-15
repomirrorworker stdout | 2022-06-03 12:51:03,769 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:51:03,769 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:51:03,770 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:51:03,779 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:51:03,779 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:51:03,779 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:51:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:51:32,454 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:51:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:51:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:52:03.733973+00:00 (in 29.998897 seconds)
repomirrorworker stdout | 2022-06-03 12:51:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:52:03 UTC)" (scheduled at 2022-06-03 12:51:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:51:33,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:51:33,751 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:51:33,754 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 51, 33, 753614), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 51, 33, 753614), 0, 2, datetime.datetime(2022, 6, 3, 12, 51, 33, 753614), True, 11, 15])
repomirrorworker stdout | 2022-06-03 12:51:33,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 12:51:33,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 51, 33, 759200), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 51, 33, 759200), 0, 2, datetime.datetime(2022, 6, 3, 12, 51, 33, 759200), True, 7, 11])
repomirrorworker stdout | 2022-06-03 12:51:33,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 12:51:33,764 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 51, 33, 763059), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 51, 33, 763059), 0, 2, datetime.datetime(2022, 6, 3, 12, 51, 33, 763059), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:51:33,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:51:33,767 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 51, 33, 766509), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 51, 33, 766509), 0, 2, datetime.datetime(2022, 6, 3, 12, 51, 33, 766509), True, 5, 9])
repomirrorworker stdout | 2022-06-03 12:51:33,769 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 12:51:33,769 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:51:33,770 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:51:33,770 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:51:33,779 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:51:33,779 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:51:33,779 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:52:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:52:02,516 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:52:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:52:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:52:33.733973+00:00 (in 29.998974 seconds)
repomirrorworker stdout | 2022-06-03 12:52:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:52:33 UTC)" (scheduled at 2022-06-03 12:52:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:52:03,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:52:03,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:52:03,751 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 52, 3, 749859), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 52, 3, 749859), 0, 2, datetime.datetime(2022, 6, 3, 12, 52, 3, 749859), True, 4, 8])
repomirrorworker stdout | 2022-06-03 12:52:03,754 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 4-8
repomirrorworker stdout | 2022-06-03 12:52:03,755 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 52, 3, 754606), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 52, 3, 754606), 0, 2, datetime.datetime(2022, 6, 3, 12, 52, 3, 754606), True, 11, 15])
repomirrorworker stdout | 2022-06-03 12:52:03,757 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 12:52:03,758 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 52, 3, 757932), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 52, 3, 757932), 0, 2, datetime.datetime(2022, 6, 3, 12, 52, 3, 757932), True, 8, 12])
repomirrorworker stdout | 2022-06-03 12:52:03,761 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 12:52:03,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 52, 3, 761111), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 52, 3, 761111), 0, 2, datetime.datetime(2022, 6, 3, 12, 52, 3, 761111), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:52:03,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:52:03,764 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:52:03,764 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:52:03,764 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:52:03,772 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:52:03,772 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:52:03,772 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:52:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:52:32,571 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:52:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:52:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:53:03.733973+00:00 (in 29.999087 seconds)
repomirrorworker stdout | 2022-06-03 12:52:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:53:03 UTC)" (scheduled at 2022-06-03 12:52:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:52:33,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:52:33,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:52:33,753 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 52, 33, 752219), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 52, 33, 752219), 0, 2, datetime.datetime(2022, 6, 3, 12, 52, 33, 752219), True, 5, 9])
repomirrorworker stdout | 2022-06-03 12:52:33,757 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 12:52:33,758 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 52, 33, 757304), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 52, 33, 757304), 0, 2, datetime.datetime(2022, 6, 3, 12, 52, 33, 757304), True, 10, 14])
repomirrorworker stdout | 2022-06-03 12:52:33,760 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 10-14
repomirrorworker stdout | 2022-06-03 12:52:33,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 52, 33, 761191), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 52, 33, 761191), 0, 2, datetime.datetime(2022, 6, 3, 12, 52, 33, 761191), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:52:33,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:52:33,765 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 52, 33, 764461), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 52, 33, 764461), 0, 2, datetime.datetime(2022, 6, 3, 12, 52, 33, 764461), True, 9, 13])
repomirrorworker stdout | 2022-06-03 12:52:33,767 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 12:52:33,768 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 52, 33, 767632), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 52, 33, 767632), 0, 2, datetime.datetime(2022, 6, 3, 12, 52, 33, 767632), True, 14, 15])
repomirrorworker stdout | 2022-06-03 12:52:33,771 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 14-15
repomirrorworker stdout | 2022-06-03 12:52:33,771 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:52:33,771 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:52:33,772 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:52:33,781 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:52:33,781 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:52:33,781 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:53:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:53:02,643 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:53:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:53:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:53:33.733973+00:00 (in 29.999029 seconds)
repomirrorworker stdout | 2022-06-03 12:53:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:53:33 UTC)" (scheduled at 2022-06-03 12:53:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:53:03,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:53:03,750 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:53:03,753 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 53, 3, 752154), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 53, 3, 752154), 0, 2, datetime.datetime(2022, 6, 3, 12, 53, 3, 752154), True, 5, 9])
repomirrorworker stdout | 2022-06-03 12:53:03,757 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 12:53:03,758 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 53, 3, 757192), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 53, 3, 757192), 0, 2, datetime.datetime(2022, 6, 3, 12, 53, 3, 757192), True, 11, 15])
repomirrorworker stdout | 2022-06-03 12:53:03,760 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 12:53:03,761 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 53, 3, 760630), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 53, 3, 760630), 0, 2, datetime.datetime(2022, 6, 3, 12, 53, 3, 760630), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:53:03,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:53:03,765 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 53, 3, 764121), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 53, 3, 764121), 0, 2, datetime.datetime(2022, 6, 3, 12, 53, 3, 764121), True, 9, 13])
repomirrorworker stdout | 2022-06-03 12:53:03,767 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 12:53:03,767 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:53:03,767 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:53:03,767 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:53:03,777 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:53:03,777 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:53:03,777 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:53:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:53:32,678 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:53:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:53:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:54:03.733973+00:00 (in 29.999055 seconds)
repomirrorworker stdout | 2022-06-03 12:53:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:54:03 UTC)" (scheduled at 2022-06-03 12:53:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:53:33,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:53:33,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:53:33,752 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 53, 33, 751738), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 53, 33, 751738), 0, 2, datetime.datetime(2022, 6, 3, 12, 53, 33, 751738), True, 4, 8])
repomirrorworker stdout | 2022-06-03 12:53:33,756 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 4-8
repomirrorworker stdout | 2022-06-03 12:53:33,757 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 53, 33, 756867), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 53, 33, 756867), 0, 2, datetime.datetime(2022, 6, 3, 12, 53, 33, 756867), True, 10, 14])
repomirrorworker stdout | 2022-06-03 12:53:33,760 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 10-14
repomirrorworker stdout | 2022-06-03 12:53:33,761 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 53, 33, 760397), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 53, 33, 760397), 0, 2, datetime.datetime(2022, 6, 3, 12, 53, 33, 760397), True, 8, 12])
repomirrorworker stdout | 2022-06-03 12:53:33,763 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 12:53:33,764 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 53, 33, 764068), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 53, 33, 764068), 0, 2, datetime.datetime(2022, 6, 3, 12, 53, 33, 764068), True, 14, 15])
repomirrorworker stdout | 2022-06-03 12:53:33,767 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 14-15
repomirrorworker stdout | 2022-06-03 12:53:33,769 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 53, 33, 768032), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 53, 33, 768032), 0, 2, datetime.datetime(2022, 6, 3, 12, 53, 33, 768032), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:53:33,775 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:53:33,775 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:53:33,775 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:53:33,776 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:53:33,784 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:53:33,784 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:53:33,784 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:54:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:54:02,725 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:54:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:54:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:54:33.733973+00:00 (in 29.999209 seconds)
repomirrorworker stdout | 2022-06-03 12:54:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:54:33 UTC)" (scheduled at 2022-06-03 12:54:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:54:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:54:03,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:54:03,752 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 54, 3, 751393), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 54, 3, 751393), 0, 2, datetime.datetime(2022, 6, 3, 12, 54, 3, 751393), True, 3, 7])
repomirrorworker stdout | 2022-06-03 12:54:03,756 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 3-7
repomirrorworker stdout | 2022-06-03 12:54:03,758 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 54, 3, 757036), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 54, 3, 757036), 0, 2, datetime.datetime(2022, 6, 3, 12, 54, 3, 757036), True, 11, 15])
repomirrorworker stdout | 2022-06-03 12:54:03,761 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 12:54:03,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 54, 3, 761247), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 54, 3, 761247), 0, 2, datetime.datetime(2022, 6, 3, 12, 54, 3, 761247), True, 7, 11])
repomirrorworker stdout | 2022-06-03 12:54:03,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 12:54:03,766 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 54, 3, 765067), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 54, 3, 765067), 0, 2, datetime.datetime(2022, 6, 3, 12, 54, 3, 765067), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:54:03,768 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:54:03,768 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:54:03,768 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:54:03,768 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:54:03,781 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:54:03,781 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:54:03,782 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:54:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:54:32,757 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:54:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:54:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:55:03.733973+00:00 (in 29.998999 seconds)
repomirrorworker stdout | 2022-06-03 12:54:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:55:03 UTC)" (scheduled at 2022-06-03 12:54:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:54:33,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:54:33,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:54:33,752 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 54, 33, 751400), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 54, 33, 751400), 0, 2, datetime.datetime(2022, 6, 3, 12, 54, 33, 751400), True, 8, 12])
repomirrorworker stdout | 2022-06-03 12:54:33,756 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 12:54:33,757 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 54, 33, 756530), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 54, 33, 756530), 0, 2, datetime.datetime(2022, 6, 3, 12, 54, 33, 756530), True, 12, 15])
repomirrorworker stdout | 2022-06-03 12:54:33,777 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 12-15
repomirrorworker stdout | 2022-06-03 12:54:33,778 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 54, 33, 777436), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 54, 33, 777436), 0, 2, datetime.datetime(2022, 6, 3, 12, 54, 33, 777436), True, 2, 6])
repomirrorworker stdout | 2022-06-03 12:54:33,781 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 12:54:33,782 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 54, 33, 781679), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 54, 33, 781679), 0, 2, datetime.datetime(2022, 6, 3, 12, 54, 33, 781679), True, 6, 10])
repomirrorworker stdout | 2022-06-03 12:54:33,785 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 12:54:33,786 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 54, 33, 785802), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 54, 33, 785802), 0, 2, datetime.datetime(2022, 6, 3, 12, 54, 33, 785802), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:54:33,789 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:54:33,789 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:54:33,789 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:54:33,789 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:54:33,798 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:54:33,798 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:54:33,798 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:55:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:55:02,815 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:55:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:55:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:55:33.733973+00:00 (in 29.998962 seconds)
repomirrorworker stdout | 2022-06-03 12:55:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:55:33 UTC)" (scheduled at 2022-06-03 12:55:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:55:03,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:55:03,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:55:03,753 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 55, 3, 751693), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 55, 3, 751693), 0, 2, datetime.datetime(2022, 6, 3, 12, 55, 3, 751693), True, 6, 10])
repomirrorworker stdout | 2022-06-03 12:55:03,758 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 12:55:03,759 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 55, 3, 758483), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 55, 3, 758483), 0, 2, datetime.datetime(2022, 6, 3, 12, 55, 3, 758483), True, 10, 14])
repomirrorworker stdout | 2022-06-03 12:55:03,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 10-14
repomirrorworker stdout | 2022-06-03 12:55:03,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 55, 3, 762940), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 55, 3, 762940), 0, 2, datetime.datetime(2022, 6, 3, 12, 55, 3, 762940), True, 14, 15])
repomirrorworker stdout | 2022-06-03 12:55:03,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 14-15
repomirrorworker stdout | 2022-06-03 12:55:03,767 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 55, 3, 767036), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 55, 3, 767036), 0, 2, datetime.datetime(2022, 6, 3, 12, 55, 3, 767036), True, 2, 6])
repomirrorworker stdout | 2022-06-03 12:55:03,770 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 12:55:03,772 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 55, 3, 771088), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 55, 3, 771088), 0, 2, datetime.datetime(2022, 6, 3, 12, 55, 3, 771088), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:55:03,774 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:55:03,774 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:55:03,774 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:55:03,775 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:55:03,785 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:55:03,785 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:55:03,785 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:55:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:55:32,859 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:55:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:55:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:56:03.733973+00:00 (in 29.999063 seconds)
repomirrorworker stdout | 2022-06-03 12:55:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:56:03 UTC)" (scheduled at 2022-06-03 12:55:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:55:33,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:55:33,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:55:33,753 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 55, 33, 752006), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 55, 33, 752006), 0, 2, datetime.datetime(2022, 6, 3, 12, 55, 33, 752006), True, 6, 10])
repomirrorworker stdout | 2022-06-03 12:55:33,757 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 12:55:33,758 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 55, 33, 757193), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 55, 33, 757193), 0, 2, datetime.datetime(2022, 6, 3, 12, 55, 33, 757193), True, 11, 15])
repomirrorworker stdout | 2022-06-03 12:55:33,760 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 12:55:33,761 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 55, 33, 760963), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 55, 33, 760963), 0, 2, datetime.datetime(2022, 6, 3, 12, 55, 33, 760963), True, 10, 14])
repomirrorworker stdout | 2022-06-03 12:55:33,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 10-14
repomirrorworker stdout | 2022-06-03 12:55:33,765 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 55, 33, 764803), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 55, 33, 764803), 0, 2, datetime.datetime(2022, 6, 3, 12, 55, 33, 764803), True, 2, 6])
repomirrorworker stdout | 2022-06-03 12:55:33,768 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 12:55:33,769 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 55, 33, 768653), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 55, 33, 768653), 0, 2, datetime.datetime(2022, 6, 3, 12, 55, 33, 768653), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:55:33,772 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:55:33,772 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:55:33,772 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:55:33,773 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:55:33,783 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:55:33,783 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:55:33,783 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:56:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:56:02,918 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:56:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:56:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:56:33.733973+00:00 (in 29.999153 seconds)
repomirrorworker stdout | 2022-06-03 12:56:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:56:33 UTC)" (scheduled at 2022-06-03 12:56:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:56:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:56:03,750 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:56:03,756 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 56, 3, 755085), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 56, 3, 755085), 0, 2, datetime.datetime(2022, 6, 3, 12, 56, 3, 755085), True, 9, 13])
repomirrorworker stdout | 2022-06-03 12:56:03,760 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 12:56:03,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 56, 3, 760769), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 56, 3, 760769), 0, 2, datetime.datetime(2022, 6, 3, 12, 56, 3, 760769), True, 13, 15])
repomirrorworker stdout | 2022-06-03 12:56:03,765 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 13-15
repomirrorworker stdout | 2022-06-03 12:56:03,766 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 56, 3, 765280), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 56, 3, 765280), 0, 2, datetime.datetime(2022, 6, 3, 12, 56, 3, 765280), True, 3, 7])
repomirrorworker stdout | 2022-06-03 12:56:03,769 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 3-7
repomirrorworker stdout | 2022-06-03 12:56:03,771 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 56, 3, 770035), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 56, 3, 770035), 0, 2, datetime.datetime(2022, 6, 3, 12, 56, 3, 770035), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:56:03,773 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:56:03,774 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 56, 3, 773695), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 56, 3, 773695), 0, 2, datetime.datetime(2022, 6, 3, 12, 56, 3, 773695), True, 7, 11])
repomirrorworker stdout | 2022-06-03 12:56:03,777 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 12:56:03,777 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:56:03,777 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:56:03,778 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:56:03,788 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:56:03,788 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:56:03,788 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:56:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:56:32,974 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:56:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:56:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:57:03.733973+00:00 (in 29.999099 seconds)
repomirrorworker stdout | 2022-06-03 12:56:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:57:03 UTC)" (scheduled at 2022-06-03 12:56:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:56:33,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:56:33,750 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:56:33,753 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 56, 33, 752607), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 56, 33, 752607), 0, 2, datetime.datetime(2022, 6, 3, 12, 56, 33, 752607), True, 9, 13])
repomirrorworker stdout | 2022-06-03 12:56:33,758 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 12:56:33,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 56, 33, 759070), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 56, 33, 759070), 0, 2, datetime.datetime(2022, 6, 3, 12, 56, 33, 759070), True, 4, 8])
repomirrorworker stdout | 2022-06-03 12:56:33,763 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 4-8
repomirrorworker stdout | 2022-06-03 12:56:33,764 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 56, 33, 763291), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 56, 33, 763291), 0, 2, datetime.datetime(2022, 6, 3, 12, 56, 33, 763291), True, 8, 12])
repomirrorworker stdout | 2022-06-03 12:56:33,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 12:56:33,768 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 56, 33, 767093), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 56, 33, 767093), 0, 2, datetime.datetime(2022, 6, 3, 12, 56, 33, 767093), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:56:33,770 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:56:33,771 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 56, 33, 770872), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 56, 33, 770872), 0, 2, datetime.datetime(2022, 6, 3, 12, 56, 33, 770872), True, 13, 15])
repomirrorworker stdout | 2022-06-03 12:56:33,775 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 13-15
repomirrorworker stdout | 2022-06-03 12:56:33,775 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:56:33,775 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:56:33,775 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:56:33,784 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:56:33,784 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:56:33,784 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:57:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:57:03,031 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:57:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:57:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:57:33.733973+00:00 (in 29.999127 seconds)
repomirrorworker stdout | 2022-06-03 12:57:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:57:33 UTC)" (scheduled at 2022-06-03 12:57:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:57:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:57:03,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:57:03,752 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 57, 3, 751453), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 57, 3, 751453), 0, 2, datetime.datetime(2022, 6, 3, 12, 57, 3, 751453), True, 9, 13])
repomirrorworker stdout | 2022-06-03 12:57:03,756 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 12:57:03,758 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 57, 3, 757152), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 57, 3, 757152), 0, 2, datetime.datetime(2022, 6, 3, 12, 57, 3, 757152), True, 13, 15])
repomirrorworker stdout | 2022-06-03 12:57:03,760 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 13-15
repomirrorworker stdout | 2022-06-03 12:57:03,761 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 57, 3, 760981), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 57, 3, 760981), 0, 2, datetime.datetime(2022, 6, 3, 12, 57, 3, 760981), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:57:03,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:57:03,765 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 57, 3, 764540), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 57, 3, 764540), 0, 2, datetime.datetime(2022, 6, 3, 12, 57, 3, 764540), True, 5, 9])
repomirrorworker stdout | 2022-06-03 12:57:03,768 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 12:57:03,768 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:57:03,768 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:57:03,769 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:57:03,778 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:57:03,778 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:57:03,778 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:57:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:57:33,070 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:57:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:57:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:58:03.733973+00:00 (in 29.999106 seconds)
repomirrorworker stdout | 2022-06-03 12:57:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:58:03 UTC)" (scheduled at 2022-06-03 12:57:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:57:33,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:57:33,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:57:33,751 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 57, 33, 750670), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 57, 33, 750670), 0, 2, datetime.datetime(2022, 6, 3, 12, 57, 33, 750670), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:57:33,755 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:57:33,756 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 57, 33, 755845), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 57, 33, 755845), 0, 2, datetime.datetime(2022, 6, 3, 12, 57, 33, 755845), True, 8, 12])
repomirrorworker stdout | 2022-06-03 12:57:33,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 12:57:33,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 57, 33, 759445), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 57, 33, 759445), 0, 2, datetime.datetime(2022, 6, 3, 12, 57, 33, 759445), True, 5, 9])
repomirrorworker stdout | 2022-06-03 12:57:33,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 12:57:33,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 57, 33, 762918), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 57, 33, 762918), 0, 2, datetime.datetime(2022, 6, 3, 12, 57, 33, 762918), True, 12, 15])
repomirrorworker stdout | 2022-06-03 12:57:33,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 12-15
repomirrorworker stdout | 2022-06-03 12:57:33,766 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:57:33,766 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:57:33,766 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:57:33,775 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:57:33,775 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:57:33,775 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:58:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:58:03,143 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:58:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:58:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:58:33.733973+00:00 (in 29.998959 seconds)
repomirrorworker stdout | 2022-06-03 12:58:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:58:33 UTC)" (scheduled at 2022-06-03 12:58:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:58:03,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:58:03,754 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:58:03,757 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 58, 3, 756440), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 58, 3, 756440), 0, 2, datetime.datetime(2022, 6, 3, 12, 58, 3, 756440), True, 11, 15])
repomirrorworker stdout | 2022-06-03 12:58:03,761 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 12:58:03,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 58, 3, 762037), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 58, 3, 762037), 0, 2, datetime.datetime(2022, 6, 3, 12, 58, 3, 762037), True, 2, 6])
repomirrorworker stdout | 2022-06-03 12:58:03,765 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 12:58:03,766 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 58, 3, 765584), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 58, 3, 765584), 0, 2, datetime.datetime(2022, 6, 3, 12, 58, 3, 765584), True, 6, 10])
repomirrorworker stdout | 2022-06-03 12:58:03,768 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 12:58:03,769 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 58, 3, 769002), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 58, 3, 769002), 0, 2, datetime.datetime(2022, 6, 3, 12, 58, 3, 769002), True, 10, 14])
repomirrorworker stdout | 2022-06-03 12:58:03,772 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 10-14
repomirrorworker stdout | 2022-06-03 12:58:03,773 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 58, 3, 772860), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 58, 3, 772860), 0, 2, datetime.datetime(2022, 6, 3, 12, 58, 3, 772860), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:58:03,776 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:58:03,776 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:58:03,776 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:58:03,776 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:58:03,785 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:58:03,785 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:58:03,785 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:58:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:58:33,190 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:58:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:58:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:59:03.733973+00:00 (in 29.998990 seconds)
repomirrorworker stdout | 2022-06-03 12:58:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:59:03 UTC)" (scheduled at 2022-06-03 12:58:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:58:33,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:58:33,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:58:33,752 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 58, 33, 751139), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 58, 33, 751139), 0, 2, datetime.datetime(2022, 6, 3, 12, 58, 33, 751139), True, 7, 11])
repomirrorworker stdout | 2022-06-03 12:58:33,756 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 12:58:33,757 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 58, 33, 756266), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 58, 33, 756266), 0, 2, datetime.datetime(2022, 6, 3, 12, 58, 33, 756266), True, 2, 6])
repomirrorworker stdout | 2022-06-03 12:58:33,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 12:58:33,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 58, 33, 759766), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 58, 33, 759766), 0, 2, datetime.datetime(2022, 6, 3, 12, 58, 33, 759766), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:58:33,777 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:58:33,778 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 58, 33, 777452), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 58, 33, 777452), 0, 2, datetime.datetime(2022, 6, 3, 12, 58, 33, 777452), True, 6, 10])
repomirrorworker stdout | 2022-06-03 12:58:33,781 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 12:58:33,782 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 58, 33, 781670), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 58, 33, 781670), 0, 2, datetime.datetime(2022, 6, 3, 12, 58, 33, 781670), True, 11, 15])
repomirrorworker stdout | 2022-06-03 12:58:33,785 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 12:58:33,785 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:58:33,785 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:58:33,785 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:58:33,793 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:58:33,793 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:58:33,794 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:59:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:59:03,246 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:59:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:59:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 12:59:33.733973+00:00 (in 29.999004 seconds)
repomirrorworker stdout | 2022-06-03 12:59:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:59:33 UTC)" (scheduled at 2022-06-03 12:59:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:59:03,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:59:03,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:59:03,751 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 59, 3, 750813), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 59, 3, 750813), 0, 2, datetime.datetime(2022, 6, 3, 12, 59, 3, 750813), True, 8, 12])
repomirrorworker stdout | 2022-06-03 12:59:03,755 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 12:59:03,756 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 59, 3, 755611), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 59, 3, 755611), 0, 2, datetime.datetime(2022, 6, 3, 12, 59, 3, 755611), True, 12, 15])
repomirrorworker stdout | 2022-06-03 12:59:03,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 12-15
repomirrorworker stdout | 2022-06-03 12:59:03,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 59, 3, 759365), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 59, 3, 759365), 0, 2, datetime.datetime(2022, 6, 3, 12, 59, 3, 759365), True, 4, 8])
repomirrorworker stdout | 2022-06-03 12:59:03,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 4-8
repomirrorworker stdout | 2022-06-03 12:59:03,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 59, 3, 762666), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 59, 3, 762666), 0, 2, datetime.datetime(2022, 6, 3, 12, 59, 3, 762666), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:59:03,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:59:03,766 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:59:03,766 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:59:03,766 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:59:03,776 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:59:03,776 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:59:03,776 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 12:59:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 12:59:33,301 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 12:59:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 12:59:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:00:03.733973+00:00 (in 29.999156 seconds)
repomirrorworker stdout | 2022-06-03 12:59:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:00:03 UTC)" (scheduled at 2022-06-03 12:59:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 12:59:33,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:59:33,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:59:33,751 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 59, 33, 750845), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 59, 33, 750845), 0, 2, datetime.datetime(2022, 6, 3, 12, 59, 33, 750845), True, 8, 12])
repomirrorworker stdout | 2022-06-03 12:59:33,755 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 12:59:33,756 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 59, 33, 755855), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 59, 33, 755855), 0, 2, datetime.datetime(2022, 6, 3, 12, 59, 33, 755855), True, 12, 15])
repomirrorworker stdout | 2022-06-03 12:59:33,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 12-15
repomirrorworker stdout | 2022-06-03 12:59:33,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 59, 33, 759236), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 59, 33, 759236), 0, 2, datetime.datetime(2022, 6, 3, 12, 59, 33, 759236), True, 1, 5])
repomirrorworker stdout | 2022-06-03 12:59:33,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 12:59:33,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 12, 59, 33, 762536), 0, 2, None, True, datetime.datetime(2022, 6, 3, 12, 59, 33, 762536), 0, 2, datetime.datetime(2022, 6, 3, 12, 59, 33, 762536), True, 5, 9])
repomirrorworker stdout | 2022-06-03 12:59:33,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 12:59:33,766 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 12:59:33,766 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:59:33,766 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 12:59:33,774 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 12:59:33,774 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 12:59:33,774 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:00:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:00:03,361 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:00:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:00:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:00:33.733973+00:00 (in 29.999079 seconds)
repomirrorworker stdout | 2022-06-03 13:00:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:00:33 UTC)" (scheduled at 2022-06-03 13:00:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:00:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:00:03,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:00:03,752 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 0, 3, 751693), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 0, 3, 751693), 0, 2, datetime.datetime(2022, 6, 3, 13, 0, 3, 751693), True, 6, 10])
repomirrorworker stdout | 2022-06-03 13:00:03,756 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 13:00:03,757 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 0, 3, 756945), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 0, 3, 756945), 0, 2, datetime.datetime(2022, 6, 3, 13, 0, 3, 756945), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:00:03,760 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:00:03,761 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 0, 3, 760284), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 0, 3, 760284), 0, 2, datetime.datetime(2022, 6, 3, 13, 0, 3, 760284), True, 5, 9])
repomirrorworker stdout | 2022-06-03 13:00:03,763 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 13:00:03,764 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 0, 3, 763245), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 0, 3, 763245), 0, 2, datetime.datetime(2022, 6, 3, 13, 0, 3, 763245), True, 11, 15])
repomirrorworker stdout | 2022-06-03 13:00:03,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 13:00:03,767 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 0, 3, 766304), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 0, 3, 766304), 0, 2, datetime.datetime(2022, 6, 3, 13, 0, 3, 766304), True, 10, 14])
repomirrorworker stdout | 2022-06-03 13:00:03,769 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 10-14
repomirrorworker stdout | 2022-06-03 13:00:03,769 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:00:03,769 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:00:03,770 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:00:03,779 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:00:03,779 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:00:03,779 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:00:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:00:33,407 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:00:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:00:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:01:03.733973+00:00 (in 29.998951 seconds)
repomirrorworker stdout | 2022-06-03 13:00:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:01:03 UTC)" (scheduled at 2022-06-03 13:00:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:00:33,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:00:33,759 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:00:33,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 0, 33, 761726), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 0, 33, 761726), 0, 2, datetime.datetime(2022, 6, 3, 13, 0, 33, 761726), True, 9, 13])
repomirrorworker stdout | 2022-06-03 13:00:33,767 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 13:00:33,768 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 0, 33, 767326), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 0, 33, 767326), 0, 2, datetime.datetime(2022, 6, 3, 13, 0, 33, 767326), True, 13, 15])
repomirrorworker stdout | 2022-06-03 13:00:33,771 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 13-15
repomirrorworker stdout | 2022-06-03 13:00:33,772 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 0, 33, 771286), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 0, 33, 771286), 0, 2, datetime.datetime(2022, 6, 3, 13, 0, 33, 771286), True, 3, 7])
repomirrorworker stdout | 2022-06-03 13:00:33,775 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 3-7
repomirrorworker stdout | 2022-06-03 13:00:33,776 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 0, 33, 775471), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 0, 33, 775471), 0, 2, datetime.datetime(2022, 6, 3, 13, 0, 33, 775471), True, 7, 11])
repomirrorworker stdout | 2022-06-03 13:00:33,779 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 13:00:33,780 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 0, 33, 779268), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 0, 33, 779268), 0, 2, datetime.datetime(2022, 6, 3, 13, 0, 33, 779268), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:00:33,783 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:00:33,783 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:00:33,783 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:00:33,783 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:00:33,793 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:00:33,793 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:00:33,794 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:01:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:01:03,465 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:01:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:01:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:01:33.733973+00:00 (in 29.998937 seconds)
repomirrorworker stdout | 2022-06-03 13:01:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:01:33 UTC)" (scheduled at 2022-06-03 13:01:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:01:03,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:01:03,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:01:03,753 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 1, 3, 751794), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 1, 3, 751794), 0, 2, datetime.datetime(2022, 6, 3, 13, 1, 3, 751794), True, 6, 10])
repomirrorworker stdout | 2022-06-03 13:01:03,757 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 13:01:03,759 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 1, 3, 757971), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 1, 3, 757971), 0, 2, datetime.datetime(2022, 6, 3, 13, 1, 3, 757971), True, 2, 6])
repomirrorworker stdout | 2022-06-03 13:01:03,761 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 13:01:03,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 1, 3, 762253), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 1, 3, 762253), 0, 2, datetime.datetime(2022, 6, 3, 13, 1, 3, 762253), True, 10, 14])
repomirrorworker stdout | 2022-06-03 13:01:03,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 10-14
repomirrorworker stdout | 2022-06-03 13:01:03,767 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 1, 3, 766136), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 1, 3, 766136), 0, 2, datetime.datetime(2022, 6, 3, 13, 1, 3, 766136), True, 14, 15])
repomirrorworker stdout | 2022-06-03 13:01:03,769 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 14-15
repomirrorworker stdout | 2022-06-03 13:01:03,770 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 1, 3, 769719), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 1, 3, 769719), 0, 2, datetime.datetime(2022, 6, 3, 13, 1, 3, 769719), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:01:03,772 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:01:03,773 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:01:03,773 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:01:03,773 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:01:03,782 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:01:03,782 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:01:03,783 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:01:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:01:33,492 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:01:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:01:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:02:03.733973+00:00 (in 29.998991 seconds)
repomirrorworker stdout | 2022-06-03 13:01:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:02:03 UTC)" (scheduled at 2022-06-03 13:01:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:01:33,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:01:33,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:01:33,753 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 1, 33, 752274), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 1, 33, 752274), 0, 2, datetime.datetime(2022, 6, 3, 13, 1, 33, 752274), True, 2, 6])
repomirrorworker stdout | 2022-06-03 13:01:33,757 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 13:01:33,758 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 1, 33, 757636), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 1, 33, 757636), 0, 2, datetime.datetime(2022, 6, 3, 13, 1, 33, 757636), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:01:33,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:01:33,764 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 1, 33, 763088), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 1, 33, 763088), 0, 2, datetime.datetime(2022, 6, 3, 13, 1, 33, 763088), True, 8, 12])
repomirrorworker stdout | 2022-06-03 13:01:33,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 13:01:33,767 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 1, 33, 766910), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 1, 33, 766910), 0, 2, datetime.datetime(2022, 6, 3, 13, 1, 33, 766910), True, 6, 10])
repomirrorworker stdout | 2022-06-03 13:01:33,770 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 13:01:33,771 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 1, 33, 770749), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 1, 33, 770749), 0, 2, datetime.datetime(2022, 6, 3, 13, 1, 33, 770749), True, 12, 15])
repomirrorworker stdout | 2022-06-03 13:01:33,774 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 12-15
repomirrorworker stdout | 2022-06-03 13:01:33,774 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:01:33,774 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:01:33,774 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:01:33,785 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:01:33,785 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:01:33,785 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:02:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:02:03,536 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:02:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:02:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:02:33.733973+00:00 (in 29.999095 seconds)
repomirrorworker stdout | 2022-06-03 13:02:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:02:33 UTC)" (scheduled at 2022-06-03 13:02:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:02:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:02:03,750 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:02:03,753 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 2, 3, 752759), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 2, 3, 752759), 0, 2, datetime.datetime(2022, 6, 3, 13, 2, 3, 752759), True, 8, 12])
repomirrorworker stdout | 2022-06-03 13:02:03,758 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 13:02:03,759 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 2, 3, 758187), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 2, 3, 758187), 0, 2, datetime.datetime(2022, 6, 3, 13, 2, 3, 758187), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:02:03,761 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:02:03,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 2, 3, 761721), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 2, 3, 761721), 0, 2, datetime.datetime(2022, 6, 3, 13, 2, 3, 761721), True, 12, 15])
repomirrorworker stdout | 2022-06-03 13:02:03,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 12-15
repomirrorworker stdout | 2022-06-03 13:02:03,765 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 2, 3, 764841), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 2, 3, 764841), 0, 2, datetime.datetime(2022, 6, 3, 13, 2, 3, 764841), True, 5, 9])
repomirrorworker stdout | 2022-06-03 13:02:03,768 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 13:02:03,768 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:02:03,768 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:02:03,768 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:02:03,779 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:02:03,779 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:02:03,779 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:02:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:02:33,605 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:02:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:02:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:03:03.733973+00:00 (in 29.999220 seconds)
repomirrorworker stdout | 2022-06-03 13:02:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:03:03 UTC)" (scheduled at 2022-06-03 13:02:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:02:33,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:02:33,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:02:33,751 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 2, 33, 750477), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 2, 33, 750477), 0, 2, datetime.datetime(2022, 6, 3, 13, 2, 33, 750477), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:02:33,755 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:02:33,756 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 2, 33, 755659), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 2, 33, 755659), 0, 2, datetime.datetime(2022, 6, 3, 13, 2, 33, 755659), True, 9, 13])
repomirrorworker stdout | 2022-06-03 13:02:33,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 13:02:33,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 2, 33, 759240), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 2, 33, 759240), 0, 2, datetime.datetime(2022, 6, 3, 13, 2, 33, 759240), True, 13, 15])
repomirrorworker stdout | 2022-06-03 13:02:33,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 13-15
repomirrorworker stdout | 2022-06-03 13:02:33,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 2, 33, 762643), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 2, 33, 762643), 0, 2, datetime.datetime(2022, 6, 3, 13, 2, 33, 762643), True, 5, 9])
repomirrorworker stdout | 2022-06-03 13:02:33,765 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 13:02:33,765 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:02:33,766 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:02:33,766 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:02:33,775 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:02:33,775 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:02:33,776 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:03:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:03:03,650 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:03:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:03:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:03:33.733973+00:00 (in 29.999057 seconds)
repomirrorworker stdout | 2022-06-03 13:03:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:03:33 UTC)" (scheduled at 2022-06-03 13:03:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:03:03,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:03:03,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:03:03,751 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 3, 3, 750374), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 3, 3, 750374), 0, 2, datetime.datetime(2022, 6, 3, 13, 3, 3, 750374), True, 7, 11])
repomirrorworker stdout | 2022-06-03 13:03:03,755 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 13:03:03,756 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 3, 3, 755436), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 3, 3, 755436), 0, 2, datetime.datetime(2022, 6, 3, 13, 3, 3, 755436), True, 11, 15])
repomirrorworker stdout | 2022-06-03 13:03:03,758 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 13:03:03,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 3, 3, 759083), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 3, 3, 759083), 0, 2, datetime.datetime(2022, 6, 3, 13, 3, 3, 759083), True, 2, 6])
repomirrorworker stdout | 2022-06-03 13:03:03,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 13:03:03,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 3, 3, 762502), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 3, 3, 762502), 0, 2, datetime.datetime(2022, 6, 3, 13, 3, 3, 762502), True, 6, 10])
repomirrorworker stdout | 2022-06-03 13:03:03,765 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 13:03:03,767 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 3, 3, 766059), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 3, 3, 766059), 0, 2, datetime.datetime(2022, 6, 3, 13, 3, 3, 766059), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:03:03,769 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:03:03,769 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:03:03,769 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:03:03,769 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:03:03,780 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:03:03,780 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:03:03,780 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:03:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:03:33,711 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:03:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:03:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:04:03.733973+00:00 (in 29.999165 seconds)
repomirrorworker stdout | 2022-06-03 13:03:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:04:03 UTC)" (scheduled at 2022-06-03 13:03:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:03:33,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:03:33,757 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:03:33,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 3, 33, 759022), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 3, 33, 759022), 0, 2, datetime.datetime(2022, 6, 3, 13, 3, 33, 759022), True, 11, 15])
repomirrorworker stdout | 2022-06-03 13:03:33,763 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 13:03:33,765 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 3, 33, 763988), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 3, 33, 763988), 0, 2, datetime.datetime(2022, 6, 3, 13, 3, 33, 763988), True, 4, 8])
repomirrorworker stdout | 2022-06-03 13:03:33,767 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 4-8
repomirrorworker stdout | 2022-06-03 13:03:33,768 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 3, 33, 767741), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 3, 33, 767741), 0, 2, datetime.datetime(2022, 6, 3, 13, 3, 33, 767741), True, 8, 12])
repomirrorworker stdout | 2022-06-03 13:03:33,771 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 13:03:33,772 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 3, 33, 771243), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 3, 33, 771243), 0, 2, datetime.datetime(2022, 6, 3, 13, 3, 33, 771243), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:03:33,774 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:03:33,774 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:03:33,774 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:03:33,774 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:03:33,783 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:03:33,783 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:03:33,784 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:04:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:04:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:04:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:04:33.733973+00:00 (in 29.999160 seconds)
repomirrorworker stdout | 2022-06-03 13:04:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:04:33 UTC)" (scheduled at 2022-06-03 13:04:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:04:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:04:03,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:04:03,751 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 4, 3, 750681), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 4, 3, 750681), 0, 2, datetime.datetime(2022, 6, 3, 13, 4, 3, 750681), True, 7, 11])
repomirrorworker stdout | 2022-06-03 13:04:03,755 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 13:04:03,756 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 4, 3, 755840), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 4, 3, 755840), 0, 2, datetime.datetime(2022, 6, 3, 13, 4, 3, 755840), True, 11, 15])
repomirrorworker stdout | 2022-06-03 13:04:03,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 13:04:03,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 4, 3, 759599), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 4, 3, 759599), 0, 2, datetime.datetime(2022, 6, 3, 13, 4, 3, 759599), True, 3, 7])
repomirrorworker stdout | 2022-06-03 13:04:03,763 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 3-7
repomirrorworker stdout | 2022-06-03 13:04:03,764 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 4, 3, 763288), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 4, 3, 763288), 0, 2, datetime.datetime(2022, 6, 3, 13, 4, 3, 763288), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:04:03,768 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:04:03,768 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:04:03,768 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:04:03,768 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:04:03,768 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:04:03,778 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:04:03,778 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:04:03,778 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:04:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:04:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:04:33,734 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:05:03.733973+00:00 (in 29.999230 seconds)
repomirrorworker stdout | 2022-06-03 13:04:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:05:03 UTC)" (scheduled at 2022-06-03 13:04:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:04:33,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:04:33,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:04:33,752 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 4, 33, 751293), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 4, 33, 751293), 0, 2, datetime.datetime(2022, 6, 3, 13, 4, 33, 751293), True, 5, 9])
repomirrorworker stdout | 2022-06-03 13:04:33,756 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 13:04:33,758 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 4, 33, 756924), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 4, 33, 756924), 0, 2, datetime.datetime(2022, 6, 3, 13, 4, 33, 756924), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:04:33,761 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:04:33,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 4, 33, 761130), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 4, 33, 761130), 0, 2, datetime.datetime(2022, 6, 3, 13, 4, 33, 761130), True, 9, 13])
repomirrorworker stdout | 2022-06-03 13:04:33,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 13:04:33,765 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 4, 33, 764944), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 4, 33, 764944), 0, 2, datetime.datetime(2022, 6, 3, 13, 4, 33, 764944), True, 13, 15])
repomirrorworker stdout | 2022-06-03 13:04:33,769 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 13-15
repomirrorworker stdout | 2022-06-03 13:04:33,769 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:04:33,769 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:04:33,769 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:04:33,780 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:04:33,780 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:04:33,780 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:05:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:04:33,821 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:05:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:05:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:05:33.733973+00:00 (in 29.999185 seconds)
repomirrorworker stdout | 2022-06-03 13:05:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:05:33 UTC)" (scheduled at 2022-06-03 13:05:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:05:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:05:03,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:05:03,752 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 5, 3, 750648), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 5, 3, 750648), 0, 2, datetime.datetime(2022, 6, 3, 13, 5, 3, 750648), True, 9, 13])
repomirrorworker stdout | 2022-06-03 13:05:03,755 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 13:05:03,757 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 5, 3, 756052), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 5, 3, 756052), 0, 2, datetime.datetime(2022, 6, 3, 13, 5, 3, 756052), True, 4, 8])
repomirrorworker stdout | 2022-06-03 13:05:03,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 4-8
repomirrorworker stdout | 2022-06-03 13:05:03,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 5, 3, 759889), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 5, 3, 759889), 0, 2, datetime.datetime(2022, 6, 3, 13, 5, 3, 759889), True, 8, 12])
repomirrorworker stdout | 2022-06-03 13:05:03,763 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 13:05:03,764 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 5, 3, 763445), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 5, 3, 763445), 0, 2, datetime.datetime(2022, 6, 3, 13, 5, 3, 763445), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:05:03,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:05:03,767 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 5, 3, 766694), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 5, 3, 766694), 0, 2, datetime.datetime(2022, 6, 3, 13, 5, 3, 766694), True, 13, 15])
repomirrorworker stdout | 2022-06-03 13:05:03,769 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 13-15
repomirrorworker stdout | 2022-06-03 13:05:03,769 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:05:03,769 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:05:03,770 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:05:03,778 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:05:03,778 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:05:03,778 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:05:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:05:03,854 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:05:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:05:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:06:03.733973+00:00 (in 29.999120 seconds)
repomirrorworker stdout | 2022-06-03 13:05:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:06:03 UTC)" (scheduled at 2022-06-03 13:05:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:05:33,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:05:33,750 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:05:33,753 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 5, 33, 752620), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 5, 33, 752620), 0, 2, datetime.datetime(2022, 6, 3, 13, 5, 33, 752620), True, 2, 6])
repomirrorworker stdout | 2022-06-03 13:05:33,757 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 13:05:33,759 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 5, 33, 757964), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 5, 33, 757964), 0, 2, datetime.datetime(2022, 6, 3, 13, 5, 33, 757964), True, 9, 13])
repomirrorworker stdout | 2022-06-03 13:05:33,761 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 13:05:33,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 5, 33, 761967), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 5, 33, 761967), 0, 2, datetime.datetime(2022, 6, 3, 13, 5, 33, 761967), True, 6, 10])
repomirrorworker stdout | 2022-06-03 13:05:33,765 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 13:05:33,766 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 5, 33, 765938), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 5, 33, 765938), 0, 2, datetime.datetime(2022, 6, 3, 13, 5, 33, 765938), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:05:33,769 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:05:33,770 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 5, 33, 769380), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 5, 33, 769380), 0, 2, datetime.datetime(2022, 6, 3, 13, 5, 33, 769380), True, 13, 15])
repomirrorworker stdout | 2022-06-03 13:05:33,773 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 13-15
repomirrorworker stdout | 2022-06-03 13:05:33,773 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:05:33,773 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:05:33,773 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:05:33,782 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:05:33,782 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:05:33,782 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:06:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:05:33,928 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:06:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:06:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:06:33.733973+00:00 (in 29.999126 seconds)
repomirrorworker stdout | 2022-06-03 13:06:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:06:33 UTC)" (scheduled at 2022-06-03 13:06:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:06:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:06:03,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:06:03,752 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 6, 3, 751008), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 6, 3, 751008), 0, 2, datetime.datetime(2022, 6, 3, 13, 6, 3, 751008), True, 3, 7])
repomirrorworker stdout | 2022-06-03 13:06:03,755 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 3-7
repomirrorworker stdout | 2022-06-03 13:06:03,757 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 6, 3, 756026), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 6, 3, 756026), 0, 2, datetime.datetime(2022, 6, 3, 13, 6, 3, 756026), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:06:03,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:06:03,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 6, 3, 759656), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 6, 3, 759656), 0, 2, datetime.datetime(2022, 6, 3, 13, 6, 3, 759656), True, 11, 15])
repomirrorworker stdout | 2022-06-03 13:06:03,763 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 13:06:03,764 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 6, 3, 763089), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 6, 3, 763089), 0, 2, datetime.datetime(2022, 6, 3, 13, 6, 3, 763089), True, 7, 11])
repomirrorworker stdout | 2022-06-03 13:06:03,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 13:06:03,766 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:06:03,766 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:06:03,766 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:06:03,775 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:06:03,775 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:06:03,775 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:06:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:06:03,982 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:06:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:06:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:07:03.733973+00:00 (in 29.999045 seconds)
repomirrorworker stdout | 2022-06-03 13:06:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:07:03 UTC)" (scheduled at 2022-06-03 13:06:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:06:33,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:06:33,750 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:06:33,753 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 6, 33, 751939), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 6, 33, 751939), 0, 2, datetime.datetime(2022, 6, 3, 13, 6, 33, 751939), True, 4, 8])
repomirrorworker stdout | 2022-06-03 13:06:33,757 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 4-8
repomirrorworker stdout | 2022-06-03 13:06:33,758 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 6, 33, 757403), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 6, 33, 757403), 0, 2, datetime.datetime(2022, 6, 3, 13, 6, 33, 757403), True, 9, 13])
repomirrorworker stdout | 2022-06-03 13:06:33,761 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 13:06:33,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 6, 33, 761293), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 6, 33, 761293), 0, 2, datetime.datetime(2022, 6, 3, 13, 6, 33, 761293), True, 8, 12])
repomirrorworker stdout | 2022-06-03 13:06:33,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 13:06:33,766 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 6, 33, 765114), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 6, 33, 765114), 0, 2, datetime.datetime(2022, 6, 3, 13, 6, 33, 765114), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:06:33,768 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:06:33,770 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 6, 33, 769209), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 6, 33, 769209), 0, 2, datetime.datetime(2022, 6, 3, 13, 6, 33, 769209), True, 13, 15])
repomirrorworker stdout | 2022-06-03 13:06:33,772 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 13-15
repomirrorworker stdout | 2022-06-03 13:06:33,772 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:06:33,772 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:06:33,773 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:06:33,786 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:06:33,786 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:06:33,786 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:07:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:06:34,047 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:07:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:07:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:07:33.733973+00:00 (in 29.998713 seconds)
repomirrorworker stdout | 2022-06-03 13:07:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:07:33 UTC)" (scheduled at 2022-06-03 13:07:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:07:03,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:07:03,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:07:03,751 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 7, 3, 750665), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 7, 3, 750665), 0, 2, datetime.datetime(2022, 6, 3, 13, 7, 3, 750665), True, 6, 10])
repomirrorworker stdout | 2022-06-03 13:07:03,755 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 13:07:03,756 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 7, 3, 755764), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 7, 3, 755764), 0, 2, datetime.datetime(2022, 6, 3, 13, 7, 3, 755764), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:07:03,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:07:03,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 7, 3, 759135), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 7, 3, 759135), 0, 2, datetime.datetime(2022, 6, 3, 13, 7, 3, 759135), True, 5, 9])
repomirrorworker stdout | 2022-06-03 13:07:03,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 13:07:03,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 7, 3, 762685), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 7, 3, 762685), 0, 2, datetime.datetime(2022, 6, 3, 13, 7, 3, 762685), True, 10, 14])
repomirrorworker stdout | 2022-06-03 13:07:03,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 10-14
repomirrorworker stdout | 2022-06-03 13:07:03,767 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 7, 3, 766205), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 7, 3, 766205), 0, 2, datetime.datetime(2022, 6, 3, 13, 7, 3, 766205), True, 14, 15])
repomirrorworker stdout | 2022-06-03 13:07:03,769 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 14-15
repomirrorworker stdout | 2022-06-03 13:07:03,769 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:07:03,769 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:07:03,769 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:07:03,779 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:07:03,779 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:07:03,779 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:07:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:07:04,104 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:07:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:07:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:08:03.733973+00:00 (in 29.999007 seconds)
repomirrorworker stdout | 2022-06-03 13:07:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:08:03 UTC)" (scheduled at 2022-06-03 13:07:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:07:33,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:07:33,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:07:33,752 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 7, 33, 751486), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 7, 33, 751486), 0, 2, datetime.datetime(2022, 6, 3, 13, 7, 33, 751486), True, 5, 9])
repomirrorworker stdout | 2022-06-03 13:07:33,761 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 13:07:33,762 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 7, 33, 761517), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 7, 33, 761517), 0, 2, datetime.datetime(2022, 6, 3, 13, 7, 33, 761517), True, 11, 15])
repomirrorworker stdout | 2022-06-03 13:07:33,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 13:07:33,765 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 7, 33, 765000), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 7, 33, 765000), 0, 2, datetime.datetime(2022, 6, 3, 13, 7, 33, 765000), True, 9, 13])
repomirrorworker stdout | 2022-06-03 13:07:33,768 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 13:07:33,769 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 7, 33, 768255), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 7, 33, 768255), 0, 2, datetime.datetime(2022, 6, 3, 13, 7, 33, 768255), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:07:33,771 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:07:33,771 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:07:33,771 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:07:33,772 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:07:33,782 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:07:33,782 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:07:33,782 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:08:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:07:34,163 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:08:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:08:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:08:33.733973+00:00 (in 29.998986 seconds)
repomirrorworker stdout | 2022-06-03 13:08:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:08:33 UTC)" (scheduled at 2022-06-03 13:08:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:08:03,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:08:03,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:08:03,752 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 8, 3, 751234), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 8, 3, 751234), 0, 2, datetime.datetime(2022, 6, 3, 13, 8, 3, 751234), True, 9, 13])
repomirrorworker stdout | 2022-06-03 13:08:03,756 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 13:08:03,757 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 8, 3, 756353), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 8, 3, 756353), 0, 2, datetime.datetime(2022, 6, 3, 13, 8, 3, 756353), True, 13, 15])
repomirrorworker stdout | 2022-06-03 13:08:03,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 13-15
repomirrorworker stdout | 2022-06-03 13:08:03,761 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 8, 3, 760046), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 8, 3, 760046), 0, 2, datetime.datetime(2022, 6, 3, 13, 8, 3, 760046), True, 3, 7])
repomirrorworker stdout | 2022-06-03 13:08:03,763 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 3-7
repomirrorworker stdout | 2022-06-03 13:08:03,764 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 8, 3, 763435), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 8, 3, 763435), 0, 2, datetime.datetime(2022, 6, 3, 13, 8, 3, 763435), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:08:03,767 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:08:03,769 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 8, 3, 767396), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 8, 3, 767396), 0, 2, datetime.datetime(2022, 6, 3, 13, 8, 3, 767396), True, 7, 11])
repomirrorworker stdout | 2022-06-03 13:08:03,771 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 13:08:03,771 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:08:03,771 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:08:03,772 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:08:03,780 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:08:03,780 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:08:03,780 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:08:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:08:04,207 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:08:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:08:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:09:03.733973+00:00 (in 29.999223 seconds)
repomirrorworker stdout | 2022-06-03 13:08:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:09:03 UTC)" (scheduled at 2022-06-03 13:08:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:08:33,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:08:33,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:08:33,751 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 8, 33, 750557), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 8, 33, 750557), 0, 2, datetime.datetime(2022, 6, 3, 13, 8, 33, 750557), True, 10, 14])
repomirrorworker stdout | 2022-06-03 13:08:33,755 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 10-14
repomirrorworker stdout | 2022-06-03 13:08:33,757 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 8, 33, 756034), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 8, 33, 756034), 0, 2, datetime.datetime(2022, 6, 3, 13, 8, 33, 756034), True, 14, 15])
repomirrorworker stdout | 2022-06-03 13:08:33,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 14-15
repomirrorworker stdout | 2022-06-03 13:08:33,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 8, 33, 759483), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 8, 33, 759483), 0, 2, datetime.datetime(2022, 6, 3, 13, 8, 33, 759483), True, 3, 7])
repomirrorworker stdout | 2022-06-03 13:08:33,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 3-7
repomirrorworker stdout | 2022-06-03 13:08:33,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 8, 33, 762973), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 8, 33, 762973), 0, 2, datetime.datetime(2022, 6, 3, 13, 8, 33, 762973), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:08:33,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:08:33,767 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 8, 33, 766499), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 8, 33, 766499), 0, 2, datetime.datetime(2022, 6, 3, 13, 8, 33, 766499), True, 7, 11])
repomirrorworker stdout | 2022-06-03 13:08:33,769 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 13:08:33,769 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:08:33,769 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:08:33,770 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:08:33,778 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:08:33,778 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:08:33,778 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:09:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:08:34,264 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:09:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:09:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:09:33.733973+00:00 (in 29.999052 seconds)
repomirrorworker stdout | 2022-06-03 13:09:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:09:33 UTC)" (scheduled at 2022-06-03 13:09:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:09:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:09:03,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:09:03,752 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 9, 3, 751213), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 9, 3, 751213), 0, 2, datetime.datetime(2022, 6, 3, 13, 9, 3, 751213), True, 11, 15])
repomirrorworker stdout | 2022-06-03 13:09:03,756 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 13:09:03,757 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 9, 3, 756229), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 9, 3, 756229), 0, 2, datetime.datetime(2022, 6, 3, 13, 9, 3, 756229), True, 2, 6])
repomirrorworker stdout | 2022-06-03 13:09:03,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 13:09:03,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 9, 3, 759761), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 9, 3, 759761), 0, 2, datetime.datetime(2022, 6, 3, 13, 9, 3, 759761), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:09:03,763 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:09:03,764 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 9, 3, 763537), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 9, 3, 763537), 0, 2, datetime.datetime(2022, 6, 3, 13, 9, 3, 763537), True, 7, 11])
repomirrorworker stdout | 2022-06-03 13:09:03,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 13:09:03,768 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 9, 3, 767027), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 9, 3, 767027), 0, 2, datetime.datetime(2022, 6, 3, 13, 9, 3, 767027), True, 6, 10])
repomirrorworker stdout | 2022-06-03 13:09:03,770 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 13:09:03,770 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:09:03,770 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:09:03,771 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:09:03,791 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:09:03,791 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:09:03,791 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:09:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:09:04,304 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:09:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:09:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:10:03.733973+00:00 (in 29.999169 seconds)
repomirrorworker stdout | 2022-06-03 13:09:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:10:03 UTC)" (scheduled at 2022-06-03 13:09:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:09:33,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:09:33,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:09:33,751 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 9, 33, 750714), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 9, 33, 750714), 0, 2, datetime.datetime(2022, 6, 3, 13, 9, 33, 750714), True, 2, 6])
repomirrorworker stdout | 2022-06-03 13:09:33,755 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 13:09:33,757 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 9, 33, 756080), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 9, 33, 756080), 0, 2, datetime.datetime(2022, 6, 3, 13, 9, 33, 756080), True, 7, 11])
repomirrorworker stdout | 2022-06-03 13:09:33,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 13:09:33,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 9, 33, 759550), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 9, 33, 759550), 0, 2, datetime.datetime(2022, 6, 3, 13, 9, 33, 759550), True, 6, 10])
repomirrorworker stdout | 2022-06-03 13:09:33,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 13:09:33,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 9, 33, 763016), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 9, 33, 763016), 0, 2, datetime.datetime(2022, 6, 3, 13, 9, 33, 763016), True, 11, 15])
repomirrorworker stdout | 2022-06-03 13:09:33,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 13:09:33,767 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 9, 33, 766621), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 9, 33, 766621), 0, 2, datetime.datetime(2022, 6, 3, 13, 9, 33, 766621), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:09:33,770 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:09:33,770 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:09:33,770 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:09:33,770 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:09:33,778 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:09:33,778 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:09:33,778 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:10:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:09:34,364 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:10:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:10:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:10:33.733973+00:00 (in 29.999195 seconds)
repomirrorworker stdout | 2022-06-03 13:10:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:10:33 UTC)" (scheduled at 2022-06-03 13:10:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:10:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:10:03,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:10:03,751 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 10, 3, 750456), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 10, 3, 750456), 0, 2, datetime.datetime(2022, 6, 3, 13, 10, 3, 750456), True, 9, 13])
repomirrorworker stdout | 2022-06-03 13:10:03,755 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 13:10:03,756 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 10, 3, 755960), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 10, 3, 755960), 0, 2, datetime.datetime(2022, 6, 3, 13, 10, 3, 755960), True, 13, 15])
repomirrorworker stdout | 2022-06-03 13:10:03,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 13-15
repomirrorworker stdout | 2022-06-03 13:10:03,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 10, 3, 759527), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 10, 3, 759527), 0, 2, datetime.datetime(2022, 6, 3, 13, 10, 3, 759527), True, 4, 8])
repomirrorworker stdout | 2022-06-03 13:10:03,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 4-8
repomirrorworker stdout | 2022-06-03 13:10:03,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 10, 3, 762986), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 10, 3, 762986), 0, 2, datetime.datetime(2022, 6, 3, 13, 10, 3, 762986), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:10:03,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:10:03,768 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 10, 3, 767019), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 10, 3, 767019), 0, 2, datetime.datetime(2022, 6, 3, 13, 10, 3, 767019), True, 8, 12])
repomirrorworker stdout | 2022-06-03 13:10:03,770 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 13:10:03,770 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:10:03,770 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:10:03,770 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:10:03,779 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:10:03,779 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:10:03,780 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:10:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:10:04,424 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:10:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:10:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:11:03.733973+00:00 (in 29.999123 seconds)
repomirrorworker stdout | 2022-06-03 13:10:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:11:03 UTC)" (scheduled at 2022-06-03 13:10:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:10:33,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:10:33,747 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:10:33,751 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 10, 33, 749865), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 10, 33, 749865), 0, 2, datetime.datetime(2022, 6, 3, 13, 10, 33, 749865), True, 10, 14])
repomirrorworker stdout | 2022-06-03 13:10:33,755 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 10-14
repomirrorworker stdout | 2022-06-03 13:10:33,756 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 10, 33, 755129), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 10, 33, 755129), 0, 2, datetime.datetime(2022, 6, 3, 13, 10, 33, 755129), True, 2, 6])
repomirrorworker stdout | 2022-06-03 13:10:33,758 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 13:10:33,759 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 10, 33, 758769), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 10, 33, 758769), 0, 2, datetime.datetime(2022, 6, 3, 13, 10, 33, 758769), True, 6, 10])
repomirrorworker stdout | 2022-06-03 13:10:33,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 13:10:33,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 10, 33, 762123), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 10, 33, 762123), 0, 2, datetime.datetime(2022, 6, 3, 13, 10, 33, 762123), True, 14, 15])
repomirrorworker stdout | 2022-06-03 13:10:33,765 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 14-15
repomirrorworker stdout | 2022-06-03 13:10:33,766 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 10, 33, 765463), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 10, 33, 765463), 0, 2, datetime.datetime(2022, 6, 3, 13, 10, 33, 765463), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:10:33,768 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:10:33,768 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:10:33,768 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:10:33,769 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:10:33,777 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:10:33,777 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:10:33,777 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:11:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:10:34,465 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:11:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:11:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:11:33.733973+00:00 (in 29.999115 seconds)
repomirrorworker stdout | 2022-06-03 13:11:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:11:33 UTC)" (scheduled at 2022-06-03 13:11:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:11:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:11:03,755 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:11:03,758 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 11, 3, 757352), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 11, 3, 757352), 0, 2, datetime.datetime(2022, 6, 3, 13, 11, 3, 757352), True, 7, 11])
repomirrorworker stdout | 2022-06-03 13:11:03,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 13:11:03,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 11, 3, 762508), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 11, 3, 762508), 0, 2, datetime.datetime(2022, 6, 3, 13, 11, 3, 762508), True, 11, 15])
repomirrorworker stdout | 2022-06-03 13:11:03,770 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 13:11:03,771 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 11, 3, 770154), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 11, 3, 770154), 0, 2, datetime.datetime(2022, 6, 3, 13, 11, 3, 770154), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:11:03,773 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:11:03,774 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 11, 3, 773666), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 11, 3, 773666), 0, 2, datetime.datetime(2022, 6, 3, 13, 11, 3, 773666), True, 5, 9])
repomirrorworker stdout | 2022-06-03 13:11:03,777 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 13:11:03,777 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:11:03,777 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:11:03,777 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:11:03,786 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:11:03,787 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:11:03,787 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:11:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:11:04,519 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:11:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:11:33,734 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:12:03.733973+00:00 (in 29.999270 seconds)
repomirrorworker stdout | 2022-06-03 13:11:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:12:03 UTC)" (scheduled at 2022-06-03 13:11:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:11:33,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:11:33,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:11:33,753 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 11, 33, 751976), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 11, 33, 751976), 0, 2, datetime.datetime(2022, 6, 3, 13, 11, 33, 751976), True, 9, 13])
repomirrorworker stdout | 2022-06-03 13:11:33,757 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 13:11:33,759 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 11, 33, 757977), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 11, 33, 757977), 0, 2, datetime.datetime(2022, 6, 3, 13, 11, 33, 757977), True, 13, 15])
repomirrorworker stdout | 2022-06-03 13:11:33,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 13-15
repomirrorworker stdout | 2022-06-03 13:11:33,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 11, 33, 762763), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 11, 33, 762763), 0, 2, datetime.datetime(2022, 6, 3, 13, 11, 33, 762763), True, 4, 8])
repomirrorworker stdout | 2022-06-03 13:11:33,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 4-8
repomirrorworker stdout | 2022-06-03 13:11:33,767 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 11, 33, 766432), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 11, 33, 766432), 0, 2, datetime.datetime(2022, 6, 3, 13, 11, 33, 766432), True, 8, 12])
repomirrorworker stdout | 2022-06-03 13:11:33,770 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 13:11:33,771 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 11, 33, 770768), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 11, 33, 770768), 0, 2, datetime.datetime(2022, 6, 3, 13, 11, 33, 770768), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:11:33,774 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:11:33,774 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:11:33,774 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:11:33,774 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:11:33,784 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:11:33,784 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:11:33,785 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:12:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:11:34,556 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:12:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:12:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:12:33.733973+00:00 (in 29.999182 seconds)
repomirrorworker stdout | 2022-06-03 13:12:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:12:33 UTC)" (scheduled at 2022-06-03 13:12:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:12:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:12:03,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:12:03,752 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 12, 3, 751063), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 12, 3, 751063), 0, 2, datetime.datetime(2022, 6, 3, 13, 12, 3, 751063), True, 9, 13])
repomirrorworker stdout | 2022-06-03 13:12:03,756 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 13:12:03,758 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 12, 3, 756983), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 12, 3, 756983), 0, 2, datetime.datetime(2022, 6, 3, 13, 12, 3, 756983), True, 13, 15])
repomirrorworker stdout | 2022-06-03 13:12:03,760 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 13-15
repomirrorworker stdout | 2022-06-03 13:12:03,761 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 12, 3, 760639), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 12, 3, 760639), 0, 2, datetime.datetime(2022, 6, 3, 13, 12, 3, 760639), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:12:03,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:12:03,765 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 12, 3, 764268), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 12, 3, 764268), 0, 2, datetime.datetime(2022, 6, 3, 13, 12, 3, 764268), True, 5, 9])
repomirrorworker stdout | 2022-06-03 13:12:03,767 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 13:12:03,767 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:12:03,767 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:12:03,767 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:12:03,776 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:12:03,776 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:12:03,776 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:12:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:12:04,609 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:12:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:12:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:13:03.733973+00:00 (in 29.998895 seconds)
repomirrorworker stdout | 2022-06-03 13:12:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:13:03 UTC)" (scheduled at 2022-06-03 13:12:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:12:33,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:12:33,750 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:12:33,753 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 12, 33, 752205), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 12, 33, 752205), 0, 2, datetime.datetime(2022, 6, 3, 13, 12, 33, 752205), True, 9, 13])
repomirrorworker stdout | 2022-06-03 13:12:33,757 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 13:12:33,758 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 12, 33, 757444), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 12, 33, 757444), 0, 2, datetime.datetime(2022, 6, 3, 13, 12, 33, 757444), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:12:33,760 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:12:33,761 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 12, 33, 761033), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 12, 33, 761033), 0, 2, datetime.datetime(2022, 6, 3, 13, 12, 33, 761033), True, 13, 15])
repomirrorworker stdout | 2022-06-03 13:12:33,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 13-15
repomirrorworker stdout | 2022-06-03 13:12:33,765 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 12, 33, 764479), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 12, 33, 764479), 0, 2, datetime.datetime(2022, 6, 3, 13, 12, 33, 764479), True, 5, 9])
repomirrorworker stdout | 2022-06-03 13:12:33,767 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 13:12:33,768 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:12:33,768 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:12:33,768 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:12:33,777 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:12:33,777 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:12:33,777 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:13:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:12:34,669 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:13:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:13:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:13:33.733973+00:00 (in 29.999191 seconds)
repomirrorworker stdout | 2022-06-03 13:13:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:13:33 UTC)" (scheduled at 2022-06-03 13:13:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:13:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:13:03,761 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:13:03,765 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 13, 3, 763787), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 13, 3, 763787), 0, 2, datetime.datetime(2022, 6, 3, 13, 13, 3, 763787), True, 9, 13])
repomirrorworker stdout | 2022-06-03 13:13:03,770 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 9-13
repomirrorworker stdout | 2022-06-03 13:13:03,771 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 13, 3, 770586), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 13, 3, 770586), 0, 2, datetime.datetime(2022, 6, 3, 13, 13, 3, 770586), True, 13, 15])
repomirrorworker stdout | 2022-06-03 13:13:03,774 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 13-15
repomirrorworker stdout | 2022-06-03 13:13:03,775 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 13, 3, 774317), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 13, 3, 774317), 0, 2, datetime.datetime(2022, 6, 3, 13, 13, 3, 774317), True, 4, 8])
repomirrorworker stdout | 2022-06-03 13:13:03,777 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 4-8
repomirrorworker stdout | 2022-06-03 13:13:03,778 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 13, 3, 777941), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 13, 3, 777941), 0, 2, datetime.datetime(2022, 6, 3, 13, 13, 3, 777941), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:13:03,781 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:13:03,782 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 13, 3, 781504), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 13, 3, 781504), 0, 2, datetime.datetime(2022, 6, 3, 13, 13, 3, 781504), True, 8, 12])
repomirrorworker stdout | 2022-06-03 13:13:03,785 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 13:13:03,785 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:13:03,785 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:13:03,785 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:13:03,794 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:13:03,794 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:13:03,794 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:13:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:13:04,725 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:13:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:13:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:14:03.733973+00:00 (in 29.999052 seconds)
repomirrorworker stdout | 2022-06-03 13:13:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:14:03 UTC)" (scheduled at 2022-06-03 13:13:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:13:33,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:13:33,748 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:13:33,751 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 13, 33, 750763), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 13, 33, 750763), 0, 2, datetime.datetime(2022, 6, 3, 13, 13, 33, 750763), True, 2, 6])
repomirrorworker stdout | 2022-06-03 13:13:33,764 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 13:13:33,766 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 13, 33, 765140), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 13, 33, 765140), 0, 2, datetime.datetime(2022, 6, 3, 13, 13, 33, 765140), True, 11, 15])
repomirrorworker stdout | 2022-06-03 13:13:33,768 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 13:13:33,770 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 13, 33, 768846), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 13, 33, 768846), 0, 2, datetime.datetime(2022, 6, 3, 13, 13, 33, 768846), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:13:33,772 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:13:33,774 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 13, 33, 772955), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 13, 33, 772955), 0, 2, datetime.datetime(2022, 6, 3, 13, 13, 33, 772955), True, 6, 10])
repomirrorworker stdout | 2022-06-03 13:13:33,776 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 13:13:33,777 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 13, 33, 776920), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 13, 33, 776920), 0, 2, datetime.datetime(2022, 6, 3, 13, 13, 33, 776920), True, 10, 14])
repomirrorworker stdout | 2022-06-03 13:13:33,779 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 10-14
repomirrorworker stdout | 2022-06-03 13:13:33,779 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:13:33,779 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:13:33,780 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:13:33,794 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:13:33,794 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:13:33,794 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:14:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:13:34,763 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:14:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:14:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:14:33.733973+00:00 (in 29.999151 seconds)
repomirrorworker stdout | 2022-06-03 13:14:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:14:33 UTC)" (scheduled at 2022-06-03 13:14:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:14:03,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:14:03,764 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:14:03,767 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 14, 3, 766499), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 14, 3, 766499), 0, 2, datetime.datetime(2022, 6, 3, 13, 14, 3, 766499), True, 4, 8])
repomirrorworker stdout | 2022-06-03 13:14:03,771 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 4-8
repomirrorworker stdout | 2022-06-03 13:14:03,772 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 14, 3, 771450), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 14, 3, 771450), 0, 2, datetime.datetime(2022, 6, 3, 13, 14, 3, 771450), True, 11, 15])
repomirrorworker stdout | 2022-06-03 13:14:03,783 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 13:14:03,784 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 14, 3, 783419), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 14, 3, 783419), 0, 2, datetime.datetime(2022, 6, 3, 13, 14, 3, 783419), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:14:03,786 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:14:03,787 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 14, 3, 786512), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 14, 3, 786512), 0, 2, datetime.datetime(2022, 6, 3, 13, 14, 3, 786512), True, 8, 12])
repomirrorworker stdout | 2022-06-03 13:14:03,790 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 8-12
repomirrorworker stdout | 2022-06-03 13:14:03,790 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:14:03,790 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:14:03,790 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:14:03,799 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:14:03,799 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:14:03,799 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:14:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:14:04,791 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:14:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:14:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:15:03.733973+00:00 (in 29.999013 seconds)
repomirrorworker stdout | 2022-06-03 13:14:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:15:03 UTC)" (scheduled at 2022-06-03 13:14:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:14:33,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:14:33,750 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:14:33,754 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 14, 33, 753141), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 14, 33, 753141), 0, 2, datetime.datetime(2022, 6, 3, 13, 14, 33, 753141), True, 2, 6])
repomirrorworker stdout | 2022-06-03 13:14:33,759 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 13:14:33,760 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 14, 33, 759263), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 14, 33, 759263), 0, 2, datetime.datetime(2022, 6, 3, 13, 14, 33, 759263), True, 11, 15])
repomirrorworker stdout | 2022-06-03 13:14:33,763 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 13:14:33,764 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 14, 33, 763113), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 14, 33, 763113), 0, 2, datetime.datetime(2022, 6, 3, 13, 14, 33, 763113), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:14:33,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:14:33,767 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 14, 33, 766609), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 14, 33, 766609), 0, 2, datetime.datetime(2022, 6, 3, 13, 14, 33, 766609), True, 7, 11])
repomirrorworker stdout | 2022-06-03 13:14:33,770 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 13:14:33,771 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 14, 33, 770294), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 14, 33, 770294), 0, 2, datetime.datetime(2022, 6, 3, 13, 14, 33, 770294), True, 6, 10])
repomirrorworker stdout | 2022-06-03 13:14:33,774 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 13:14:33,774 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:14:33,774 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:14:33,774 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:14:33,784 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:14:33,784 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:14:33,784 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:15:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:14:34,887 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:15:03,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:15:03,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:15:33.733973+00:00 (in 29.998993 seconds)
repomirrorworker stdout | 2022-06-03 13:15:03,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:15:33 UTC)" (scheduled at 2022-06-03 13:15:03.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:15:03,736 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:15:03,750 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:15:03,754 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 15, 3, 752667), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 15, 3, 752667), 0, 2, datetime.datetime(2022, 6, 3, 13, 15, 3, 752667), True, 10, 14])
repomirrorworker stdout | 2022-06-03 13:15:03,757 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 10-14
repomirrorworker stdout | 2022-06-03 13:15:03,759 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 15, 3, 758113), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 15, 3, 758113), 0, 2, datetime.datetime(2022, 6, 3, 13, 15, 3, 758113), True, 6, 10])
repomirrorworker stdout | 2022-06-03 13:15:03,762 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 13:15:03,763 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 15, 3, 762437), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 15, 3, 762437), 0, 2, datetime.datetime(2022, 6, 3, 13, 15, 3, 762437), True, 14, 15])
repomirrorworker stdout | 2022-06-03 13:15:03,766 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 14-15
repomirrorworker stdout | 2022-06-03 13:15:03,767 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 15, 3, 766510), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 15, 3, 766510), 0, 2, datetime.datetime(2022, 6, 3, 13, 15, 3, 766510), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:15:03,770 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:15:03,771 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 15, 3, 770301), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 15, 3, 770301), 0, 2, datetime.datetime(2022, 6, 3, 13, 15, 3, 770301), True, 5, 9])
repomirrorworker stdout | 2022-06-03 13:15:03,773 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 5-9
repomirrorworker stdout | 2022-06-03 13:15:03,774 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:15:03,774 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:15:03,774 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:15:03,788 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:15:03,788 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:15:03,788 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:15:33 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:15:04,948 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found
repomirrorworker stdout | 2022-06-03 13:15:33,734 [64] [DEBUG] [apscheduler.scheduler] Looking for jobs to run
repomirrorworker stdout | 2022-06-03 13:15:33,735 [64] [DEBUG] [apscheduler.scheduler] Next wakeup is due at 2022-06-03 13:16:03.733973+00:00 (in 29.999077 seconds)
repomirrorworker stdout | 2022-06-03 13:15:33,735 [64] [INFO] [apscheduler.executors.default] Running job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:16:03 UTC)" (scheduled at 2022-06-03 13:15:33.733973+00:00)
repomirrorworker stdout | 2022-06-03 13:15:33,735 [64] [DEBUG] [peewee] ('SELECT Min("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:15:33,749 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:15:33,753 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 15, 33, 751968), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 15, 33, 751968), 0, 2, datetime.datetime(2022, 6, 3, 13, 15, 33, 751968), True, 7, 11])
repomirrorworker stdout | 2022-06-03 13:15:33,774 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 7-11
repomirrorworker stdout | 2022-06-03 13:15:33,776 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 15, 33, 774818), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 15, 33, 774818), 0, 2, datetime.datetime(2022, 6, 3, 13, 15, 33, 774818), True, 11, 15])
repomirrorworker stdout | 2022-06-03 13:15:33,779 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 11-15
repomirrorworker stdout | 2022-06-03 13:15:33,780 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 15, 33, 779285), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 15, 33, 779285), 0, 2, datetime.datetime(2022, 6, 3, 13, 15, 33, 779285), True, 2, 6])
repomirrorworker stdout | 2022-06-03 13:15:33,783 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 2-6
repomirrorworker stdout | 2022-06-03 13:15:33,784 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 15, 33, 783446), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 15, 33, 783446), 0, 2, datetime.datetime(2022, 6, 3, 13, 15, 33, 783446), True, 1, 5])
repomirrorworker stdout | 2022-06-03 13:15:33,787 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 1-5
repomirrorworker stdout | 2022-06-03 13:15:33,788 [64] [DEBUG] [peewee] ('SELECT "t1"."id", "t1"."repository_id", "t1"."creation_date", "t1"."is_enabled", "t1"."mirror_type", "t1"."internal_robot_id", "t1"."external_reference", "t1"."external_registry_username", "t1"."external_registry_password", "t1"."external_registry_config", "t1"."sync_interval", "t1"."sync_start_date", "t1"."sync_expiration_date", "t1"."sync_retries_remaining", "t1"."sync_status", "t1"."sync_transaction_id", "t1"."root_rule_id" FROM "repomirrorconfig" AS "t1" INNER JOIN "repository" AS "t2" ON ("t1"."repository_id" = "t2"."id") WHERE (((("t2"."state" = %s) AND (((("t1"."sync_status" = %s) AND ("t1"."sync_expiration_date" IS %s)) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" != %s)) AND ("t1"."sync_expiration_date" IS %s)) AND ("t1"."is_enabled" = %s))) OR ((((("t1"."sync_start_date" <= %s) AND ("t1"."sync_retries_remaining" > %s)) AND ("t1"."sync_status" = %s)) AND ("t1"."sync_expiration_date" <= %s)) AND ("t1"."is_enabled" = %s)))) AND ("t1"."id" >= %s)) AND ("t1"."id" < %s)) ORDER BY "t1"."id"', [2, 3, None, datetime.datetime(2022, 6, 3, 13, 15, 33, 787251), 0, 2, None, True, datetime.datetime(2022, 6, 3, 13, 15, 33, 787251), 0, 2, datetime.datetime(2022, 6, 3, 13, 15, 33, 787251), True, 6, 10])
repomirrorworker stdout | 2022-06-03 13:15:33,790 [64] [INFO] [util.migrate.allocator] No candidates, marking entire block completed 6-10
repomirrorworker stdout | 2022-06-03 13:15:33,790 [64] [INFO] [util.migrate.allocator] No more work
repomirrorworker stdout | 2022-06-03 13:15:33,790 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:15:33,791 [64] [DEBUG] [peewee] ('SELECT Max("t1"."id") FROM "repomirrorconfig" AS "t1"', [])
repomirrorworker stdout | 2022-06-03 13:15:33,800 [64] [DEBUG] [workers.repomirrorworker] Found no additional repositories to mirror
repomirrorworker stdout | 2022-06-03 13:15:33,800 [64] [DEBUG] [data.database] Disconnecting from database.
repomirrorworker stdout | 2022-06-03 13:15:33,800 [64] [INFO] [apscheduler.executors.default] Job "RepoMirrorWorker._process_mirrors (trigger: interval[0:00:30], next run at: 2022-06-03 13:16:03 UTC)" executed successfully
repomirrorworker stdout | 2022-06-03 13:15:35,013 [64] [ERROR] [util.metrics.prometheus] failed to push registry to pushgateway at http://localhost:9091 with grouping key {'host': 'quay-quay-mirror-6458cdb9cc-s5wrx', 'process_name': 'repomirrorworker.py', 'pid': '64'}
repomirrorworker stdout | Traceback (most recent call last):
repomirrorworker stdout |   File "/quay-registry/util/metrics/prometheus.py", line 121, in run
repomirrorworker stdout |     push_to_gateway(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 289, in push_to_gateway
repomirrorworker stdout |     _use_gateway('PUT', gateway, job, registry, grouping_key, timeout, handler)
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 358, in _use_gateway
repomirrorworker stdout |     handler(
repomirrorworker stdout |   File "/usr/local/lib/python3.8/site-packages/prometheus_client/exposition.py", line 221, in handle
repomirrorworker stdout |     resp = build_opener(HTTPHandler).open(request, timeout=timeout)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 531, in open
repomirrorworker stdout |     response = meth(req, response)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 640, in http_response
repomirrorworker stdout |     response = self.parent.error(
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 569, in error
repomirrorworker stdout |     return self._call_chain(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 502, in _call_chain
repomirrorworker stdout |     result = func(*args)
repomirrorworker stdout |   File "/usr/lib64/python3.8/urllib/request.py", line 649, in http_error_default
repomirrorworker stdout |     raise HTTPError(req.full_url, code, msg, hdrs, fp)
repomirrorworker stdout | urllib.error.HTTPError: HTTP Error 404: Not Found